feat(server): add observability and operator controls (#16)

src/FinalFactory.Rendezvous.Server/Program.cs

@@ -5,6 +5,8 @@ using FinalFactory.Rendezvous.Server.Browser;
 using FinalFactory.Rendezvous.Server.ConnectionOutcomes;
 using FinalFactory.Rendezvous.Server.Http;
 using FinalFactory.Rendezvous.Server.JoinAttempts;
+using FinalFactory.Rendezvous.Server.Observability;
+using FinalFactory.Rendezvous.Server.Operations;
 using FinalFactory.Rendezvous.Server.Provisioning;
 using FinalFactory.Rendezvous.Server.Sessions;
 using FinalFactory.Rendezvous.Server.State;
@@ -13,6 +15,9 @@ using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.OpenApi;
 
 WebApplicationBuilder builder = WebApplication.CreateBuilder(args);
+builder.Logging.AddFilter(
+    "Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware",
+    LogLevel.None);
 bool isOpenApiGeneration = string.Equals(
     System.Reflection.Assembly.GetEntryAssembly()?.GetName().Name,
     "GetDocument.Insider",
@@ -61,6 +66,14 @@ builder.Services.AddOpenApi("v1", static options =>
             In = ParameterLocation.Header,
             Description = "Attempt-scoped client capability returned only to the joining caller.",
         };
+        const string operatorSchemeName = "OperatorBearer";
+        document.Components.SecuritySchemes[operatorSchemeName] = new OpenApiSecurityScheme
+        {
+            Type = SecuritySchemeType.Http,
+            Scheme = "bearer",
+            BearerFormat = "rv1 operator credential",
+            Description = "Operator-only credential with an explicit permission set.",
+        };
 
         HashSet<string> securedOperations = new(StringComparer.Ordinal)
         {
@@ -69,8 +82,17 @@ builder.Services.AddOpenApi("v1", static options =>
             "UpdateSession",
             "DeleteSession",
         };
+        HashSet<string> operatorOperations = new(StringComparer.Ordinal)
+        {
+            "GetOperatorStatus",
+            "RevokeOperatorListing",
+            "RevokeOperatorPrincipal",
+            "RevokeOperatorSigningKey",
+            "BeginOperatorDrain",
+        };
         OpenApiSecuritySchemeReference reference = new(schemeName, document, null);
         OpenApiSecuritySchemeReference attemptReference = new(attemptSchemeName, document, null);
+        OpenApiSecuritySchemeReference operatorReference = new(operatorSchemeName, document, null);
         foreach (OpenApiPathItem path in document.Paths.Values)
         {
             if (path.Operations is null)
@@ -99,29 +121,58 @@ builder.Services.AddOpenApi("v1", static options =>
                 });
             }
 
+            foreach (OpenApiOperation operation in path.Operations.Values.Where(
+                         operation => operatorOperations.Contains(
+                             operation.OperationId ?? string.Empty)))
+            {
+                operation.Security ??= [];
+                operation.Security.Add(new OpenApiSecurityRequirement
+                {
+                    [operatorReference] = [],
+                });
+            }
+
             foreach (OpenApiOperation operation in path.Operations.Values)
             {
-                if (operation.Responses is null
-                    || !operation.Responses.TryGetValue(
-                        StatusCodes.Status429TooManyRequests.ToString(
-                            System.Globalization.CultureInfo.InvariantCulture),
-                        out IOpenApiResponse? response)
-                    || response is not OpenApiResponse concreteResponse)
+                if (operation.Responses is null)
                 {
                     continue;
                 }
 
-                concreteResponse.Headers ??=
-                    new Dictionary<string, IOpenApiHeader>(StringComparer.OrdinalIgnoreCase);
-                concreteResponse.Headers["Retry-After"] = new OpenApiHeader
+                foreach ((string status, IOpenApiResponse response) in operation.Responses)
                 {
-                    Description = "Whole seconds before the caller should retry (1-60).",
-                    Schema = new OpenApiSchema
+                    if (response is not OpenApiResponse concreteResponse)
                     {
-                        Type = JsonSchemaType.Integer,
-                        Format = "int32",
-                    },
-                };
+                        continue;
+                    }
+
+                    concreteResponse.Headers ??=
+                        new Dictionary<string, IOpenApiHeader>(StringComparer.OrdinalIgnoreCase);
+                    concreteResponse.Headers["X-Rendezvous-Correlation-ID"] = new OpenApiHeader
+                    {
+                        Description = "Safe request correlation identifier generated by the service.",
+                        Schema = new OpenApiSchema
+                        {
+                            Type = JsonSchemaType.String,
+                        },
+                    };
+                    if (string.Equals(
+                            status,
+                            StatusCodes.Status429TooManyRequests.ToString(
+                                System.Globalization.CultureInfo.InvariantCulture),
+                            StringComparison.Ordinal))
+                    {
+                        concreteResponse.Headers["Retry-After"] = new OpenApiHeader
+                        {
+                            Description = "Whole seconds before the caller should retry (1-60).",
+                            Schema = new OpenApiSchema
+                            {
+                                Type = JsonSchemaType.Integer,
+                                Format = "int32",
+                            },
+                        };
+                    }
+                }
             }
         }
 
@@ -166,8 +217,18 @@ builder.Services
                    && addresses.All(
                        static value => IPAddress.TryParse(value, out _)),
         "Trusted proxy addresses must contain at most 32 literal IP addresses.")
+    .Validate(
+        options => options.OperatorAllowedAddresses is { Length: <= 32 } addresses
+                   && addresses.All(
+                       static value => IPAddress.TryParse(value, out _)),
+        "Operator allowed addresses must contain at most 32 literal IP addresses.")
     .ValidateOnStart();
 builder.Services.AddSingleton<AbuseProtectionService>();
+builder.Services
+    .AddOptions<AuditOptions>()
+    .BindConfiguration(AuditOptions.SectionName)
+    .ValidateDataAnnotations()
+    .ValidateOnStart();
 AbuseProtectionOptions configuredAbuseProtection = builder.Configuration
     .GetSection(AbuseProtectionOptions.SectionName)
     .Get<AbuseProtectionOptions>() ?? new AbuseProtectionOptions();
@@ -180,8 +241,13 @@ InMemoryEphemeralRendezvousStore stateStore = new(
     stateOptions,
     rendezvousClock,
     rendezvousClock);
+builder.Services.AddSingleton(stateStore);
 builder.Services.AddSingleton<IEphemeralRendezvousStore>(stateStore);
 builder.Services.AddSingleton<IWallClock>(rendezvousClock);
+builder.Services.AddSingleton<IMonotonicClock>(rendezvousClock);
+builder.Services.AddSingleton<RendezvousTelemetry>();
+builder.Services.AddSingleton<AuditTrail>();
+builder.Services.AddSingleton<RendezvousReadiness>();
 
 if (isOpenApiGeneration)
 {
@@ -214,6 +280,7 @@ else
     builder.Services.AddSingleton<JoinAttemptService>();
     builder.Services.AddSingleton<ConnectionOutcomeMetrics>();
     builder.Services.AddSingleton<ConnectionOutcomeService>();
+    builder.Services.AddSingleton<OperatorService>();
     builder.Services.AddSingleton(new ProvisioningReadiness(true));
 }
 
@@ -246,34 +313,13 @@ if (TrustedProxyForwarding.IsEnabled(configuredAbuseProtection))
 {
     app.UseForwardedHeaders();
 }
+app.UseMiddleware<TelemetryMiddleware>();
 app.UseExceptionHandler();
 app.UseMiddleware<HttpAbuseProtectionMiddleware>();
 app.MapOpenApi();
 app.MapRendezvousContractEndpoints();
-app.MapGet(
-        "/health/live",
-        static () => Results.Ok(new HealthResponse { Status = "live" }))
-    .Produces<HealthResponse>()
-    .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
-    .WithName("GetLiveness")
-    .WithTags("Health");
-app.MapGet(
-    "/health/ready",
-    static (
-        UdpMediatorService mediator,
-        ProvisioningReadiness provisioning,
-        IEphemeralRendezvousStore state) =>
-        mediator.LocalEndpoint is null
-        || !provisioning.IsReady
-        || !state.IsAvailable
-        || state.IsDraining
-        ? Results.StatusCode(StatusCodes.Status503ServiceUnavailable)
-        : Results.Ok(new HealthResponse { Status = "ready" }))
-    .Produces<HealthResponse>()
-    .Produces<ApiError>(StatusCodes.Status429TooManyRequests)
-    .Produces(StatusCodes.Status503ServiceUnavailable)
-    .WithName("GetReadiness")
-    .WithTags("Health");
+app.MapOperatorEndpoints();
+app.MapRendezvousHealthEndpoints();
 
 await app.RunAsync();