using System.Diagnostics; using System.Diagnostics.Metrics; using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using FinalFactory.Rendezvous.Contracts; using FinalFactory.Rendezvous.Server.Abuse; using FinalFactory.Rendezvous.Server.Http; using FinalFactory.Rendezvous.Server.JoinAttempts; using FinalFactory.Rendezvous.Server.Observability; using FinalFactory.Rendezvous.Server.Operations; using FinalFactory.Rendezvous.Server.Provisioning; using FinalFactory.Rendezvous.Server.Sessions; using FinalFactory.Rendezvous.Server.State; using FinalFactory.Rendezvous.Server.Transport; using FinalFactory.Rendezvous.Tests.Observability; using FinalFactory.Rendezvous.Tests.Provisioning; using FinalFactory.Rendezvous.Tests.State; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting.Server; using Microsoft.AspNetCore.Hosting.Server.Features; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace FinalFactory.Rendezvous.Tests.Operations; [Collection(RendezvousTelemetryIsolation.Name)] public sealed class OperatorEndpointTests { [Fact] public async Task OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection() { await using OperatorTestHost host = await OperatorTestHost.StartAsync(); List telemetryData = []; using MeterListener meterListener = new(); meterListener.InstrumentPublished = (instrument, listener) => { if (instrument.Meter.Name == RendezvousTelemetry.MeterName) { listener.EnableMeasurementEvents(instrument); } }; meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => CaptureMeasurement(telemetryData, instrument, value, tags)); meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => CaptureMeasurement(telemetryData, instrument, value, tags)); meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => CaptureMeasurement(telemetryData, instrument, value, tags)); meterListener.Start(); using ActivityListener activityListener = new() { ShouldListenTo = static source => source.Name == RendezvousTelemetry.ActivitySourceName, Sample = static (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, ActivityStopped = activity => { telemetryData.Add(activity.DisplayName); telemetryData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}")); }, }; ActivitySource.AddActivityListener(activityListener); using HttpResponseMessage liveBeforeDependencies = await host.Client.GetAsync("/health/live"); Assert.Equal(HttpStatusCode.OK, liveBeforeDependencies.StatusCode); using HttpResponseMessage readyBeforeUdp = await host.Client.GetAsync("/health/ready"); Assert.Equal(HttpStatusCode.ServiceUnavailable, readyBeforeUdp.StatusCode); using HttpResponseMessage unauthenticated = await host.Client.GetAsync("/v1/operator/status"); Assert.Equal(HttpStatusCode.Unauthorized, unauthenticated.StatusCode); AuthenticationHeaderValue challenge = Assert.Single( unauthenticated.Headers.WwwAuthenticate); Assert.Equal("Bearer", challenge.Scheme); Assert.Equal("realm=\"operator\"", challenge.Parameter); using HttpResponseMessage publisher = await SendAsync( host, HttpMethod.Get, "/v1/operator/status", host.PublisherCredential); Assert.Equal(HttpStatusCode.Unauthorized, publisher.StatusCode); using HttpResponseMessage status = await SendAsync( host, HttpMethod.Get, "/v1/operator/status", host.ReadOnlyOperatorCredential); Assert.Equal(HttpStatusCode.OK, status.StatusCode); string statusJson = await status.Content.ReadAsStringAsync(); Assert.Contains("not-ready", statusJson, StringComparison.Ordinal); Assert.DoesNotContain(host.OwnerCanary, statusJson, StringComparison.Ordinal); Assert.DoesNotContain("203.0.113.25", statusJson, StringComparison.Ordinal); Assert.DoesNotContain("metadata", statusJson, StringComparison.OrdinalIgnoreCase); OperatorStatusResponse? operatorStatus = JsonSerializer.Deserialize( statusJson, ContractJson.Options); Assert.Contains(operatorStatus!.Tenants, static tenant => tenant.GameId == "space-game" && tenant.EnvironmentId == "production" && tenant.Status == "enabled"); Assert.Contains(operatorStatus.SigningKeys, static key => key.KeyId == OperatorTestHost.OperatorKeyId && key.Status == "signing" && key.CredentialKinds.SequenceEqual(["Operator"])); await host.StartUdpAsync(); using HttpResponseMessage readyAfterUdp = await host.Client.GetAsync("/health/ready"); Assert.Equal(HttpStatusCode.OK, readyAfterUdp.StatusCode); using HttpResponseMessage exception = await SendAsync( host, HttpMethod.Post, "/test/exception", host.FullOperatorCredential); Assert.Equal(HttpStatusCode.InternalServerError, exception.StatusCode); using HttpResponseMessage saturatedPublic = await host.Client.GetAsync("/test/public"); Assert.Equal(HttpStatusCode.TooManyRequests, saturatedPublic.StatusCode); using HttpResponseMessage forbidden = await SendAsync( host, HttpMethod.Post, "/v1/operator/drain", host.ReadOnlyOperatorCredential, new BeginDrainRequest { Confirmation = "DRAIN" }); Assert.Equal(HttpStatusCode.Forbidden, forbidden.StatusCode); SessionListingId listingId = host.CreateListing(host.OwnerCanary); using HttpResponseMessage unconfirmedListing = await SendAsync( host, HttpMethod.Post, "/v1/operator/listings/revoke", host.FullOperatorCredential, new RevokeListingRequest { ListingId = listingId.ToString(), ConfirmListingId = Guid.NewGuid().ToString("D"), }); Assert.Equal(HttpStatusCode.BadRequest, unconfirmedListing.StatusCode); Assert.True(host.Store.GetListing(listingId, false).Succeeded); using HttpResponseMessage revokedListing = await SendAsync( host, HttpMethod.Post, "/v1/operator/listings/revoke", host.FullOperatorCredential, new RevokeListingRequest { ListingId = listingId.ToString(), ConfirmListingId = listingId.ToString(), }); Assert.Equal(HttpStatusCode.OK, revokedListing.StatusCode); Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(listingId, false).Code); const string principalCanary = "publisher-player-canary"; SessionListingId principalListing = host.CreateListing(principalCanary); using HttpResponseMessage unconfirmedPrincipal = await SendAsync( host, HttpMethod.Post, "/v1/operator/principals/revoke", host.FullOperatorCredential, new RevokePrincipalRequest { Subject = principalCanary, ConfirmSubject = "different-subject", LifetimeSeconds = 60, }); Assert.Equal(HttpStatusCode.BadRequest, unconfirmedPrincipal.StatusCode); Assert.True(host.Store.GetListing(principalListing, false).Succeeded); using HttpResponseMessage revokedPrincipal = await SendAsync( host, HttpMethod.Post, "/v1/operator/principals/revoke", host.FullOperatorCredential, new RevokePrincipalRequest { Subject = principalCanary, ConfirmSubject = principalCanary, LifetimeSeconds = 60, }); Assert.Equal(HttpStatusCode.OK, revokedPrincipal.StatusCode); OperatorActionResponse? principalResult = await revokedPrincipal.Content .ReadFromJsonAsync(ContractJson.Options); Assert.Equal(1, principalResult!.AffectedResources); Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(principalListing, false).Code); StoreResult blockedPublisher = host.CreateListingResult( principalCanary, out _); Assert.Equal(StoreResultCode.Revoked, blockedPublisher.Code); using HttpResponseMessage drain = await SendAsync( host, HttpMethod.Post, "/v1/operator/drain", host.FullOperatorCredential, new BeginDrainRequest { Confirmation = "DRAIN" }); Assert.Equal(HttpStatusCode.OK, drain.StatusCode); Assert.True(host.Store.IsDraining); using HttpResponseMessage liveDuringDrain = await host.Client.GetAsync("/health/live"); Assert.Equal(HttpStatusCode.OK, liveDuringDrain.StatusCode); using HttpResponseMessage readyDuringDrain = await host.Client.GetAsync("/health/ready"); Assert.Equal(HttpStatusCode.ServiceUnavailable, readyDuringDrain.StatusCode); using HttpResponseMessage firstPublisherKeyRevocation = await SendAsync( host, HttpMethod.Post, "/v1/operator/keys/revoke", host.FullOperatorCredential, new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" }); Assert.Equal(HttpStatusCode.OK, firstPublisherKeyRevocation.StatusCode); using HttpResponseMessage repeatedPublisherKeyRevocation = await SendAsync( host, HttpMethod.Post, "/v1/operator/keys/revoke", host.FullOperatorCredential, new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" }); Assert.Equal(HttpStatusCode.OK, repeatedPublisherKeyRevocation.StatusCode); using HttpResponseMessage unconfirmedKey = await SendAsync( host, HttpMethod.Post, "/v1/operator/keys/revoke", host.FullOperatorCredential, new RevokeSigningKeyRequest { KeyId = OperatorTestHost.OperatorKeyId, ConfirmKeyId = "different-key", }); Assert.Equal(HttpStatusCode.BadRequest, unconfirmedKey.StatusCode); using HttpResponseMessage revokedKey = await SendAsync( host, HttpMethod.Post, "/v1/operator/keys/revoke", host.FullOperatorCredential, new RevokeSigningKeyRequest { KeyId = OperatorTestHost.OperatorKeyId, ConfirmKeyId = OperatorTestHost.OperatorKeyId, }); Assert.Equal(HttpStatusCode.OK, revokedKey.StatusCode); using HttpResponseMessage afterKeyRevocation = await SendAsync( host, HttpMethod.Get, "/v1/operator/status", host.FullOperatorCredential); Assert.Equal(HttpStatusCode.Unauthorized, afterKeyRevocation.StatusCode); string auditText = string.Join('|', host.Audit.GetEntriesForTests()); string logText = string.Join('|', host.AuditLogger.Messages.Concat(host.AllLogs.Messages)); Assert.DoesNotContain(host.OwnerCanary, auditText, StringComparison.Ordinal); Assert.DoesNotContain(principalCanary, auditText, StringComparison.Ordinal); Assert.DoesNotContain(listingId.ToString(), auditText, StringComparison.Ordinal); Assert.DoesNotContain(host.OwnerCanary, logText, StringComparison.Ordinal); Assert.DoesNotContain(principalCanary, logText, StringComparison.Ordinal); Assert.DoesNotContain("exception-secret-canary", logText, StringComparison.Ordinal); Assert.DoesNotContain(host.FullOperatorCredential, logText, StringComparison.Ordinal); meterListener.RecordObservableInstruments(); string telemetryText = string.Join('|', telemetryData); Assert.DoesNotContain(host.OwnerCanary, telemetryText, StringComparison.Ordinal); Assert.DoesNotContain(principalCanary, telemetryText, StringComparison.Ordinal); Assert.DoesNotContain("exception-secret-canary", telemetryText, StringComparison.Ordinal); Assert.DoesNotContain(host.FullOperatorCredential, telemetryText, StringComparison.Ordinal); Assert.DoesNotContain(listingId.ToString(), telemetryText, StringComparison.Ordinal); Assert.DoesNotContain("203.0.113.25", telemetryText, StringComparison.Ordinal); Assert.Contains(host.Audit.GetEntriesForTests(), static entry => entry.Action == "begin-drain" && entry.Result == "succeeded"); Assert.Contains(host.Audit.GetEntriesForTests(), static entry => entry.Action == "revoke-listing" && entry.Result == "rejected"); Assert.Contains(host.Audit.GetEntriesForTests(), static entry => entry.Action == "begin-drain" && entry.Result == "forbidden"); } private static async Task SendAsync( OperatorTestHost host, HttpMethod method, string path, string bearer, object? body = null) { using HttpRequestMessage request = new(method, path); request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearer); if (body is not null) { request.Content = JsonContent.Create(body, options: ContractJson.Options); } return await host.Client.SendAsync(request); } private static void CaptureMeasurement( List destination, Instrument instrument, T value, ReadOnlySpan> tags) where T : struct { destination.Add($"{instrument.Name}={value}"); destination.AddRange(tags.ToArray().Select(static tag => $"{tag.Key}={tag.Value}")); } private sealed class OperatorTestHost : IAsyncDisposable { private readonly WebApplication _application; private int _listingSequence; private bool _udpStarted; private OperatorTestHost( WebApplication application, HttpClient client, ManualRendezvousClock clock, InMemoryEphemeralRendezvousStore store, AuditTrail audit, CapturingLogger auditLogger, CapturingLoggerProvider allLogs, string publisherCredential, string readOnlyOperatorCredential, string fullOperatorCredential) { _application = application; Client = client; Clock = clock; Store = store; Audit = audit; AuditLogger = auditLogger; AllLogs = allLogs; PublisherCredential = publisherCredential; ReadOnlyOperatorCredential = readOnlyOperatorCredential; FullOperatorCredential = fullOperatorCredential; } internal const string OperatorKeyId = "operator-key"; internal string OwnerCanary { get; } = "publisher-owner-canary"; internal HttpClient Client { get; } internal ManualRendezvousClock Clock { get; } internal InMemoryEphemeralRendezvousStore Store { get; } internal AuditTrail Audit { get; } internal CapturingLogger AuditLogger { get; } internal CapturingLoggerProvider AllLogs { get; } internal string PublisherCredential { get; } internal string ReadOnlyOperatorCredential { get; } internal string FullOperatorCredential { get; } internal static async Task StartAsync() { ManualRendezvousClock clock = new(ProvisioningTestData.Now); EphemeralStoreOptions stateOptions = new(); InMemoryEphemeralRendezvousStore store = new(stateOptions, clock, clock); SigningKeyOptions publisherKey = ProvisioningTestData.CreateKey(); SigningKeyOptions operatorKey = ProvisioningTestData.CreateKey( OperatorKeyId, "operator-secret", credentialKinds: [PrincipalCredentialKind.Operator], gameId: null, environmentId: null); ProvisioningOptions options = ProvisioningTestData.CreateOptions(); options.SigningKeys = [publisherKey, operatorKey]; ProvisioningRuntime provisioning = ProvisioningRuntime.Create( options, ProvisioningTestData.CreateSecrets("secret-1", "operator-secret"), clock.UtcNow); string publisherCredential = provisioning.Credentials.Issue( ProvisioningTestData.CreateDedicatedPublisher(), clock.UtcNow); string readOnlyCredential = provisioning.Credentials.Issue( new OperatorPrincipal( "operator-readonly", clock.UtcNow.AddMinutes(10), [OperatorPermission.ReadPolicy]), clock.UtcNow); string fullCredential = provisioning.Credentials.Issue( new OperatorPrincipal( "operator-full", clock.UtcNow.AddMinutes(10), Enum.GetValues()), clock.UtcNow); CapturingLogger auditLogger = new(); CapturingLoggerProvider allLogs = new(); EphemeralCapabilityIssuer capabilities = new(); WebApplicationBuilder builder = WebApplication.CreateBuilder(); builder.WebHost.UseUrls("http://127.0.0.1:0"); builder.Logging.ClearProviders(); builder.Logging.SetMinimumLevel(LogLevel.Debug); builder.Logging.AddProvider(allLogs); builder.Logging.AddFilter( "Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware", LogLevel.None); builder.Services.ConfigureHttpJsonOptions(static json => ContractJson.Configure(json.SerializerOptions)); builder.Services.Configure(static route => route.ThrowOnBadRequest = true); builder.Services.AddProblemDetails(); builder.Services.AddExceptionHandler(); builder.Services.AddOptions().Configure(static abuse => { abuse.OperatorAllowedAddresses = ["127.0.0.1"]; abuse.HttpGlobalRequestsPerWindow = 1; abuse.HttpOptionalRequestsPerWindow = 1; abuse.HttpIpPrefixRequestsPerWindow = 1; abuse.HttpOptionalIpPrefixRequestsPerWindow = 1; }); builder.Services.AddOptions(); builder.Services.AddOptions().Configure(static udp => { udp.ListenAddress = "127.0.0.1"; udp.Port = 0; }); builder.Services.AddSingleton(provisioning); builder.Services.AddSingleton(provisioning.Policies); builder.Services.AddSingleton(provisioning.Credentials); builder.Services.AddSingleton(provisioning.PublisherAuthorization); builder.Services.AddSingleton(store); builder.Services.AddSingleton(store); builder.Services.AddSingleton(clock); builder.Services.AddSingleton(clock); builder.Services.AddSingleton(capabilities); builder.Services.AddSingleton(capabilities); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(new ProvisioningReadiness(true)); builder.Services.AddSingleton(); builder.Services.AddSingleton>(auditLogger); builder.Services.AddSingleton(); builder.Services.AddSingleton(); WebApplication app = builder.Build(); app.UseMiddleware(); app.UseExceptionHandler(); app.UseMiddleware(); app.MapOperatorEndpoints(); app.MapRendezvousHealthEndpoints(); app.MapGet("/test/public", static () => Results.Ok()).WithName("TestPublic"); app.MapPost( "/test/exception", static IResult () => throw new InvalidOperationException("exception-secret-canary")) .WithName("TestSecretException"); await app.StartAsync(); IServer server = app.Services.GetRequiredService(); string address = Assert.Single(server.Features.Get()!.Addresses); return new OperatorTestHost( app, new HttpClient { BaseAddress = new Uri(address) }, clock, store, app.Services.GetRequiredService(), auditLogger, allLogs, publisherCredential, readOnlyCredential, fullCredential); } internal SessionListingId CreateListing(string owner) { StoreResult result = CreateListingResult(owner, out SessionListingId listingId); Assert.True(result.Succeeded); return listingId; } internal StoreResult CreateListingResult( string owner, out SessionListingId listingId) { int sequence = Interlocked.Increment(ref _listingSequence); listingId = new(Guid.NewGuid()); StoreResult result = Store.CreateListing(new( $"operator-listing-{sequence}", $"operator-request-{sequence}", new ListingDefinition { ListingId = listingId, LeaseId = new(Guid.NewGuid()), Scope = new(new GameId("space-game"), new EnvironmentId("production")), OwnerSubject = owner, RegionId = new("eu-central"), ProtocolVersion = 7, BuildVersion = "1.0.0", DisplayName = "Operator test listing", Visibility = ListingVisibility.Public, TrustMode = PublisherTrustMode.ManagedDedicated, CurrentPlayers = 1, MaximumPlayers = 4, Metadata = new Dictionary { ["mode"] = "online-coop" }, LeaseFingerprint = new("lease-fingerprint"), HostPresenceHandle = new(Guid.NewGuid()), HostPresenceFingerprint = new("presence-fingerprint"), CapabilityDerivationSalt = new string('A', 43), })); return result; } internal async Task StartUdpAsync() { await _application.Services.GetRequiredService() .StartAsync(CancellationToken.None); _udpStarted = true; } public async ValueTask DisposeAsync() { Client.Dispose(); if (_udpStarted) { await _application.Services.GetRequiredService() .StopAsync(CancellationToken.None); } await _application.StopAsync(); await _application.DisposeAsync(); } } }