feat(server): add observability and operator controls (#16)

src/FinalFactory.Rendezvous.Server/Operations/OperatorEndpoints.cs

@@ -0,0 +1,428 @@
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+using Microsoft.AspNetCore.Mvc;
+
+namespace FinalFactory.Rendezvous.Server.Operations;
+
+internal static class OperatorEndpoints
+{
+    private const string CorrelationHeader = "X-Rendezvous-Correlation-ID";
+
+    public static IEndpointRouteBuilder MapOperatorEndpoints(this IEndpointRouteBuilder endpoints)
+    {
+        RouteGroupBuilder group = endpoints.MapGroup("/v1/operator").WithTags("Operator");
+        group.MapGet("/status", GetStatus)
+            .Produces<OperatorStatusResponse>()
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("GetOperatorStatus");
+        group.MapPost("/listings/revoke", RevokeListing)
+            .Accepts<RevokeListingRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .Produces<ApiError>(StatusCodes.Status503ServiceUnavailable)
+            .WithName("RevokeOperatorListing");
+        group.MapPost("/principals/revoke", RevokePrincipal)
+            .Accepts<RevokePrincipalRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .Produces<ApiError>(StatusCodes.Status503ServiceUnavailable)
+            .WithName("RevokeOperatorPrincipal");
+        group.MapPost("/keys/revoke", RevokeSigningKey)
+            .Accepts<RevokeSigningKeyRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("RevokeOperatorSigningKey");
+        group.MapPost("/drain", BeginDrain)
+            .Accepts<BeginDrainRequest>("application/json")
+            .Produces<OperatorActionResponse>()
+            .Produces<ApiError>(StatusCodes.Status400BadRequest)
+            .Produces<ApiError>(StatusCodes.Status413PayloadTooLarge)
+            .Produces<ApiError>(StatusCodes.Status401Unauthorized)
+            .Produces<ApiError>(StatusCodes.Status403Forbidden)
+            .Produces<ApiError>(StatusCodes.Status404NotFound)
+            .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
+            .WithName("BeginOperatorDrain");
+        return endpoints;
+    }
+
+    private static IResult GetStatus(
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.ReadPolicy,
+                "inspect-status",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        OperatorStatusResponse response = service.GetStatus();
+        audit.Record(
+            principal!.Subject,
+            "inspect-status",
+            "succeeded",
+            "service",
+            "rendezvous",
+            Correlation(context));
+        return Results.Ok(response);
+    }
+
+    private static IResult RevokeListing(
+        [FromBody] RevokeListingRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context,
+        CancellationToken cancellationToken)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.RevokePublisher,
+                "revoke-listing",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        bool valid = SessionListingId.TryParse(request.ListingId, out SessionListingId listingId);
+        if (!valid || !string.Equals(request.ListingId, request.ConfirmListingId, StringComparison.Ordinal))
+        {
+            AuditRejected(audit, principal!, "revoke-listing", "listing", request.ListingId, context);
+            return BadRequest("A valid listing ID and an exact repeated confirmation are required.");
+        }
+
+        StoreResult<bool> result = service.RevokeListing(listingId, cancellationToken);
+        return StoreActionResult(
+            result.Code,
+            audit,
+            principal!,
+            "revoke-listing",
+            "listing",
+            request.ListingId,
+            context,
+            affectedResources: result.Succeeded ? 1 : null);
+    }
+
+    private static IResult RevokePrincipal(
+        [FromBody] RevokePrincipalRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context,
+        CancellationToken cancellationToken)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.RevokePublisher,
+                "revoke-principal",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        bool safeSubject = request.Subject is { Length: > 0 and <= 128 }
+            && request.Subject.All(static character => character is >= '!' and <= '~');
+        if (!safeSubject
+            || !string.Equals(request.Subject, request.ConfirmSubject, StringComparison.Ordinal)
+            || request.LifetimeSeconds is < 1 or > 600)
+        {
+            AuditRejected(audit, principal!, "revoke-principal", "principal", request.Subject, context);
+            return BadRequest("A valid subject, exact repeated confirmation, and 1-600 second lifetime are required.");
+        }
+
+        StoreResult<int> result = service.RevokePrincipal(
+            request.Subject,
+            TimeSpan.FromSeconds(request.LifetimeSeconds),
+            cancellationToken);
+        return StoreActionResult(
+            result.Code,
+            audit,
+            principal!,
+            "revoke-principal",
+            "principal",
+            request.Subject,
+            context,
+            result.Value);
+    }
+
+    private static IResult RevokeSigningKey(
+        [FromBody] RevokeSigningKeyRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.RotateKeys,
+                "revoke-signing-key",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        bool safeKeyId = request.KeyId is { Length: > 0 and <= 64 }
+            && request.KeyId.All(static character => character is
+                >= 'A' and <= 'Z'
+                or >= 'a' and <= 'z'
+                or >= '0' and <= '9'
+                or '-'
+                or '_');
+        if (!safeKeyId || !string.Equals(request.KeyId, request.ConfirmKeyId, StringComparison.Ordinal))
+        {
+            AuditRejected(audit, principal!, "revoke-signing-key", "signing-key", request.KeyId, context);
+            return BadRequest("A valid key ID and an exact repeated confirmation are required.");
+        }
+
+        bool revoked = service.RevokeSigningKey(request.KeyId);
+        string result = revoked ? "succeeded" : "not-found";
+        audit.Record(
+            principal!.Subject,
+            "revoke-signing-key",
+            result,
+            "signing-key",
+            request.KeyId,
+            Correlation(context));
+        return revoked
+            ? Results.Ok(new OperatorActionResponse { Status = "completed" })
+            : Error(RendezvousErrorCode.NotFound, "The requested resource was not found.");
+    }
+
+    private static IResult BeginDrain(
+        [FromBody] BeginDrainRequest request,
+        [FromHeader(Name = "Authorization")] string? authorization,
+        [FromServices] PrincipalCredentialService credentials,
+        [FromServices] IWallClock clock,
+        [FromServices] OperatorService service,
+        [FromServices] AuditTrail audit,
+        [FromServices] RendezvousTelemetry telemetry,
+        HttpContext context,
+        CancellationToken cancellationToken)
+    {
+        if (!TryAuthorize(
+                authorization,
+                OperatorPermission.ManagePolicy,
+                "begin-drain",
+                credentials,
+                clock,
+                audit,
+                telemetry,
+                context,
+                out OperatorPrincipal? principal,
+                out IResult? failure))
+        {
+            return failure!;
+        }
+
+        if (!string.Equals(request.Confirmation, "DRAIN", StringComparison.Ordinal))
+        {
+            AuditRejected(audit, principal!, "begin-drain", "service", "rendezvous", context);
+            return BadRequest("The confirmation value must be exactly 'DRAIN'.");
+        }
+
+        service.BeginDrain(cancellationToken);
+        audit.Record(
+            principal!.Subject,
+            "begin-drain",
+            "succeeded",
+            "service",
+            "rendezvous",
+            Correlation(context));
+        return Results.Ok(new OperatorActionResponse { Status = "draining" });
+    }
+
+    private static bool TryAuthorize(
+        string? authorization,
+        OperatorPermission requiredPermission,
+        string operation,
+        PrincipalCredentialService credentials,
+        IWallClock clock,
+        AuditTrail audit,
+        RendezvousTelemetry telemetry,
+        HttpContext context,
+        out OperatorPrincipal? principal,
+        out IResult? failure)
+    {
+        principal = null;
+        failure = null;
+        const string prefix = "Bearer ";
+        if (authorization is null
+            || !authorization.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))
+        {
+            telemetry.RecordOperatorAuthentication("rejected");
+            failure = AuthenticationRequired(context);
+            return false;
+        }
+
+        CredentialValidationResult validation = credentials.Validate(
+            authorization[prefix.Length..],
+            clock.UtcNow);
+        if (!validation.IsValid || validation.Principal is not OperatorPrincipal candidate)
+        {
+            telemetry.RecordOperatorAuthentication("rejected");
+            failure = AuthenticationRequired(context);
+            return false;
+        }
+
+        if (!candidate.Permissions.Contains(requiredPermission))
+        {
+            telemetry.RecordOperatorAuthentication("forbidden");
+            audit.Record(
+                candidate.Subject,
+                operation,
+                "forbidden",
+                "operator-operation",
+                operation,
+                Correlation(context));
+            failure = Error(RendezvousErrorCode.Forbidden, "The operator is not authorized for this operation.");
+            return false;
+        }
+
+        telemetry.RecordOperatorAuthentication("accepted");
+        principal = candidate;
+        return true;
+    }
+
+    private static IResult StoreActionResult(
+        StoreResultCode code,
+        AuditTrail audit,
+        OperatorPrincipal principal,
+        string action,
+        string targetKind,
+        string targetIdentifier,
+        HttpContext context,
+        int? affectedResources)
+    {
+        string auditResult = code == StoreResultCode.Success
+            ? "succeeded"
+            : code.ToString().ToLowerInvariant();
+        audit.Record(
+            principal.Subject,
+            action,
+            auditResult,
+            targetKind,
+            targetIdentifier,
+            Correlation(context));
+        return code switch
+        {
+            StoreResultCode.Success => Results.Ok(new OperatorActionResponse
+            {
+                Status = "completed",
+                AffectedResources = affectedResources,
+            }),
+            StoreResultCode.NotFound => Error(
+                RendezvousErrorCode.NotFound,
+                "The requested resource was not found."),
+            StoreResultCode.CapacityExceeded => Error(
+                RendezvousErrorCode.CapacityExceeded,
+                "The operation could not be retained within the configured capacity."),
+            StoreResultCode.ServiceUnavailable or StoreResultCode.Draining => Error(
+                RendezvousErrorCode.ServiceUnavailable,
+                "The service is not available for this operation."),
+            _ => Error(RendezvousErrorCode.Conflict, "The operation could not be completed."),
+        };
+    }
+
+    private static void AuditRejected(
+        AuditTrail audit,
+        OperatorPrincipal principal,
+        string action,
+        string targetKind,
+        string? targetIdentifier,
+        HttpContext context) => audit.Record(
+            principal.Subject,
+            action,
+            "rejected",
+            targetKind,
+            targetIdentifier ?? string.Empty,
+            Correlation(context));
+
+    private static string Correlation(HttpContext context) =>
+        context.Response.Headers[CorrelationHeader].ToString() is { Length: > 0 } value
+            ? value
+            : "unavailable";
+
+    private static IResult AuthenticationRequired(HttpContext context)
+    {
+        context.Response.Headers.WWWAuthenticate = "Bearer realm=\"operator\"";
+        return Error(
+            RendezvousErrorCode.AuthenticationRequired,
+            "A valid operator bearer credential is required.");
+    }
+
+    private static IResult BadRequest(string message) => Error(RendezvousErrorCode.InvalidRequest, message);
+
+    private static IResult Error(RendezvousErrorCode code, string message) => Results.Json(
+        new ApiError { Code = code, Message = message },
+        ContractJson.Options,
+        statusCode: code switch
+        {
+            RendezvousErrorCode.AuthenticationRequired => StatusCodes.Status401Unauthorized,
+            RendezvousErrorCode.Forbidden => StatusCodes.Status403Forbidden,
+            RendezvousErrorCode.NotFound => StatusCodes.Status404NotFound,
+            RendezvousErrorCode.CapacityExceeded => StatusCodes.Status429TooManyRequests,
+            RendezvousErrorCode.ServiceUnavailable => StatusCodes.Status503ServiceUnavailable,
+            RendezvousErrorCode.Conflict => StatusCodes.Status409Conflict,
+            _ => StatusCodes.Status400BadRequest,
+        });
+}

src/FinalFactory.Rendezvous.Server/Operations/OperatorModels.cs

@@ -0,0 +1,82 @@
+namespace FinalFactory.Rendezvous.Server.Operations;
+
+internal sealed record OperatorStatusResponse
+{
+    public required string Status { get; init; }
+    public required OperatorReadinessResponse Readiness { get; init; }
+    public required OperatorStoreResponse Store { get; init; }
+    public required IReadOnlyList<OperatorTenantResponse> Tenants { get; init; }
+    public required IReadOnlyList<OperatorSigningKeyResponse> SigningKeys { get; init; }
+    public required IReadOnlyDictionary<string, long> AuditCounts { get; init; }
+}
+
+internal sealed record OperatorReadinessResponse
+{
+    public required bool HttpListener { get; init; }
+    public required bool UdpIpv4Listener { get; init; }
+    public required bool UdpIpv6Listener { get; init; }
+    public required bool Provisioning { get; init; }
+    public required bool Store { get; init; }
+    public required bool Draining { get; init; }
+}
+
+internal sealed record OperatorStoreResponse
+{
+    public required int ActiveListings { get; init; }
+    public required int FreshPresenceBindings { get; init; }
+    public required int ActiveJoinAttempts { get; init; }
+    public required int RetainedOutcomeReports { get; init; }
+    public required int ReplayMarkers { get; init; }
+    public required int PrincipalRevocations { get; init; }
+    public required int IdempotencyEntries { get; init; }
+    public required long MaintenanceSweeps { get; init; }
+    public required long ExpiryChurn { get; init; }
+}
+
+internal sealed record OperatorTenantResponse
+{
+    public required string GameId { get; init; }
+    public required string EnvironmentId { get; init; }
+    public required string Status { get; init; }
+}
+
+internal sealed record OperatorSigningKeyResponse
+{
+    public required string KeyId { get; init; }
+    public required string Status { get; init; }
+    public required DateTimeOffset SignUntil { get; init; }
+    public required DateTimeOffset VerifyUntil { get; init; }
+    public string? GameId { get; init; }
+    public string? EnvironmentId { get; init; }
+    public required IReadOnlyList<string> CredentialKinds { get; init; }
+}
+
+internal sealed record OperatorActionResponse
+{
+    public required string Status { get; init; }
+    public int? AffectedResources { get; init; }
+}
+
+internal sealed record RevokeListingRequest
+{
+    public required string ListingId { get; init; }
+    public required string ConfirmListingId { get; init; }
+}
+
+internal sealed record RevokePrincipalRequest
+{
+    public required string Subject { get; init; }
+    public required string ConfirmSubject { get; init; }
+    public required int LifetimeSeconds { get; init; }
+}
+
+internal sealed record RevokeSigningKeyRequest
+{
+    public required string KeyId { get; init; }
+    public required string ConfirmKeyId { get; init; }
+}
+
+internal sealed record BeginDrainRequest
+{
+    public required string Confirmation { get; init; }
+}

src/FinalFactory.Rendezvous.Server/Operations/OperatorService.cs

@@ -0,0 +1,81 @@
+using FinalFactory.Rendezvous.Contracts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Provisioning;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Server.Operations;
+
+internal sealed class OperatorService(
+    InMemoryEphemeralRendezvousStore store,
+    ProvisioningRuntime provisioning,
+    RendezvousReadiness readiness,
+    AuditTrail audit,
+    IWallClock clock)
+{
+    public OperatorStatusResponse GetStatus()
+    {
+        ReadinessSnapshot readinessSnapshot = readiness.GetSnapshot();
+        EphemeralStoreSnapshot storeSnapshot = store.GetSnapshot();
+        return new OperatorStatusResponse
+        {
+            Status = readinessSnapshot.IsReady ? "ready" : "not-ready",
+            Readiness = new OperatorReadinessResponse
+            {
+                HttpListener = readinessSnapshot.HttpListenerReady,
+                UdpIpv4Listener = readinessSnapshot.UdpIpv4ListenerReady,
+                UdpIpv6Listener = readinessSnapshot.UdpIpv6ListenerReady,
+                Provisioning = readinessSnapshot.ProvisioningReady,
+                Store = readinessSnapshot.StoreAvailable,
+                Draining = readinessSnapshot.Draining,
+            },
+            Store = new OperatorStoreResponse
+            {
+                ActiveListings = storeSnapshot.ActiveListings,
+                FreshPresenceBindings = storeSnapshot.FreshPresenceBindings,
+                ActiveJoinAttempts = storeSnapshot.ActiveJoinAttempts,
+                RetainedOutcomeReports = storeSnapshot.RetainedOutcomeReports,
+                ReplayMarkers = storeSnapshot.ReplayMarkers,
+                PrincipalRevocations = storeSnapshot.PrincipalRevocations,
+                IdempotencyEntries = storeSnapshot.IdempotencyEntries,
+                MaintenanceSweeps = storeSnapshot.MaintenanceSweeps,
+                ExpiryChurn = storeSnapshot.ExpiryChurn,
+            },
+            Tenants = provisioning.Policies.EnabledPolicies
+                .OrderBy(static policy => policy.GameId.Value, StringComparer.Ordinal)
+                .ThenBy(static policy => policy.EnvironmentId.Value, StringComparer.Ordinal)
+                .Select(static policy => new OperatorTenantResponse
+                {
+                    GameId = policy.GameId.Value,
+                    EnvironmentId = policy.EnvironmentId.Value,
+                    Status = "enabled",
+                })
+                .ToArray(),
+            SigningKeys = provisioning.SigningKeys.GetStatuses(clock.UtcNow)
+                .Select(static key => new OperatorSigningKeyResponse
+                {
+                    KeyId = key.KeyId,
+                    Status = key.Status,
+                    SignUntil = key.SignUntil,
+                    VerifyUntil = key.VerifyUntil,
+                    GameId = key.GameId,
+                    EnvironmentId = key.EnvironmentId,
+                    CredentialKinds = key.CredentialKinds,
+                })
+                .ToArray(),
+            AuditCounts = audit.GetAggregateCounts(),
+        };
+    }
+
+    public StoreResult<bool> RevokeListing(
+        SessionListingId listingId,
+        CancellationToken cancellationToken) => store.RevokeListing(listingId, cancellationToken);
+
+    public StoreResult<int> RevokePrincipal(
+        string subject,
+        TimeSpan lifetime,
+        CancellationToken cancellationToken) => store.RevokePrincipal(subject, lifetime, cancellationToken);
+
+    public bool RevokeSigningKey(string keyId) => provisioning.SigningKeys.Revoke(keyId);
+
+    public void BeginDrain(CancellationToken cancellationToken) => store.BeginDrain(cancellationToken);
+}