feat(server): add observability and operator controls (#16)
quality-gate / quality (push) Failing after 1m1s
quality-gate / quality (push) Failing after 1m1s
This commit is contained in:
KyuubiYoru
@@ -11,6 +11,11 @@ public sealed class OpenApiCompatibilityTests
|
||||
"/v1/join-attempts",
|
||||
"/v1/join-attempts/{attemptId}",
|
||||
"/v1/join-attempts/{attemptId}/outcome",
|
||||
"/v1/operator/drain",
|
||||
"/v1/operator/keys/revoke",
|
||||
"/v1/operator/listings/revoke",
|
||||
"/v1/operator/principals/revoke",
|
||||
"/v1/operator/status",
|
||||
"/v1/sessions",
|
||||
"/v1/sessions/{listingId}",
|
||||
"/v1/sessions/{listingId}/join-attempts",
|
||||
@@ -104,6 +109,11 @@ public sealed class OpenApiCompatibilityTests
|
||||
Assert.Equal(
|
||||
"X-Rendezvous-Client-Punch-Capability",
|
||||
attemptCapability.GetProperty("name").GetString());
|
||||
JsonElement operatorBearer = root.GetProperty("components")
|
||||
.GetProperty("securitySchemes")
|
||||
.GetProperty("OperatorBearer");
|
||||
Assert.Equal("http", operatorBearer.GetProperty("type").GetString());
|
||||
Assert.Equal("bearer", operatorBearer.GetProperty("scheme").GetString());
|
||||
(string Path, string Method)[] publisherOperations =
|
||||
[
|
||||
("/v1/sessions", "post"),
|
||||
@@ -120,6 +130,23 @@ public sealed class OpenApiCompatibilityTests
|
||||
Assert.True(security[0].TryGetProperty("PublisherBearer", out _));
|
||||
}
|
||||
|
||||
(string Path, string Method)[] operatorOperations =
|
||||
[
|
||||
("/v1/operator/status", "get"),
|
||||
("/v1/operator/listings/revoke", "post"),
|
||||
("/v1/operator/principals/revoke", "post"),
|
||||
("/v1/operator/keys/revoke", "post"),
|
||||
("/v1/operator/drain", "post"),
|
||||
];
|
||||
foreach ((string operationPath, string method) in operatorOperations)
|
||||
{
|
||||
JsonElement security = root.GetProperty("paths")
|
||||
.GetProperty(operationPath)
|
||||
.GetProperty(method)
|
||||
.GetProperty("security");
|
||||
Assert.True(security[0].TryGetProperty("OperatorBearer", out _));
|
||||
}
|
||||
|
||||
JsonElement cancelParameters = root.GetProperty("paths")
|
||||
.GetProperty("/v1/join-attempts/{attemptId}")
|
||||
.GetProperty("delete")
|
||||
@@ -157,6 +184,15 @@ public sealed class OpenApiCompatibilityTests
|
||||
static item => item.Name is "get" or "post" or "put" or "delete"))
|
||||
{
|
||||
JsonElement responses = operation.Value.GetProperty("responses");
|
||||
foreach (JsonProperty response in responses.EnumerateObject())
|
||||
{
|
||||
JsonElement correlation = response.Value.GetProperty("headers")
|
||||
.GetProperty("X-Rendezvous-Correlation-ID");
|
||||
Assert.Equal(
|
||||
"string",
|
||||
correlation.GetProperty("schema").GetProperty("type").GetString());
|
||||
}
|
||||
|
||||
if (!responses.TryGetProperty("429", out JsonElement overloaded))
|
||||
{
|
||||
continue;
|
||||
@@ -171,7 +207,7 @@ public sealed class OpenApiCompatibilityTests
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(12, overloadContracts);
|
||||
Assert.Equal(17, overloadContracts);
|
||||
(string Path, string Method)[] bodyOperations =
|
||||
[
|
||||
("/v1/sessions", "post"),
|
||||
@@ -180,6 +216,10 @@ public sealed class OpenApiCompatibilityTests
|
||||
("/v1/sessions/{listingId}", "delete"),
|
||||
("/v1/join-attempts", "post"),
|
||||
("/v1/join-attempts/{attemptId}/outcome", "post"),
|
||||
("/v1/operator/listings/revoke", "post"),
|
||||
("/v1/operator/principals/revoke", "post"),
|
||||
("/v1/operator/keys/revoke", "post"),
|
||||
("/v1/operator/drain", "post"),
|
||||
];
|
||||
foreach ((string operationPath, string method) in bodyOperations)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Net;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
using FinalFactory.Rendezvous.Server.Transport;
|
||||
using FinalFactory.Rendezvous.Tests.State;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Tests.Observability;
|
||||
|
||||
[CollectionDefinition(RendezvousTelemetryIsolation.Name, DisableParallelization = true)]
|
||||
public sealed class RendezvousTelemetryIsolation
|
||||
{
|
||||
public const string Name = "Rendezvous telemetry";
|
||||
}
|
||||
|
||||
[Collection(RendezvousTelemetryIsolation.Name)]
|
||||
public sealed class ObservabilityTests
|
||||
{
|
||||
private static readonly HashSet<string> AllowedTagKeys =
|
||||
[
|
||||
"operation",
|
||||
"status_code",
|
||||
"result",
|
||||
"transport",
|
||||
"partition",
|
||||
"action",
|
||||
"outcome",
|
||||
"elapsed_bucket",
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void MetricsAndTracesUseBoundedDimensionsWithoutSensitiveValues()
|
||||
{
|
||||
EphemeralStateFixture fixture = new();
|
||||
using RendezvousTelemetry telemetry = new(fixture.Store);
|
||||
List<Measurement> measurements = [];
|
||||
using MeterListener meterListener = new();
|
||||
meterListener.InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
|
||||
{
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
meterListener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
||||
measurements.Add(new(instrument.Name, value, Tags(tags))));
|
||||
meterListener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
|
||||
measurements.Add(new(instrument.Name, value, Tags(tags))));
|
||||
meterListener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
||||
measurements.Add(new(instrument.Name, value, Tags(tags))));
|
||||
meterListener.Start();
|
||||
|
||||
Activity? observed = null;
|
||||
List<string> activityData = [];
|
||||
using ActivityListener activityListener = new()
|
||||
{
|
||||
ShouldListenTo = source => source.Name == RendezvousTelemetry.ActivitySourceName,
|
||||
Sample = static (ref ActivityCreationOptions<ActivityContext> _) =>
|
||||
ActivitySamplingResult.AllData,
|
||||
ActivityStopped = activity =>
|
||||
{
|
||||
observed = activity;
|
||||
activityData.Add(activity.DisplayName);
|
||||
activityData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}"));
|
||||
},
|
||||
};
|
||||
ActivitySource.AddActivityListener(activityListener);
|
||||
|
||||
const string secret = "secret-player-token-canary";
|
||||
using (telemetry.StartActivity("HTTP GetOperatorStatus", ActivityKind.Server))
|
||||
{
|
||||
telemetry.RecordHttp("GetOperatorStatus", 200, 3.5);
|
||||
telemetry.RecordUdp("frozen", "Introduced", 1.25);
|
||||
telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
|
||||
telemetry.RecordAudit("revoke-listing", "succeeded");
|
||||
telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
|
||||
telemetry.RecordOperatorAuthentication("accepted");
|
||||
telemetry.RecordPairingLatency(12.5);
|
||||
}
|
||||
NatMediationProcessor processor = new(null!, null!, null!, telemetry: telemetry);
|
||||
Assert.Equal(
|
||||
NatMediationResult.Dropped,
|
||||
processor.ProcessRequest(
|
||||
new IPEndPoint(IPAddress.Parse("10.0.0.8"), 9000),
|
||||
new IPEndPoint(IPAddress.Parse("203.0.113.8"), 50000),
|
||||
secret,
|
||||
NoopIntroductionSink.Instance));
|
||||
|
||||
meterListener.RecordObservableInstruments();
|
||||
Assert.NotNull(observed);
|
||||
string flattened = string.Join('|', measurements.Select(static item => item.ToString()));
|
||||
Assert.DoesNotContain(secret, flattened, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(secret, string.Join('|', activityData), StringComparison.Ordinal);
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.http.requests");
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.udp.results");
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.limiter.drops");
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.queue.depth");
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.store.active_leases");
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.store.expiry_churn");
|
||||
Assert.Contains(measurements, static item => item.Name == "rendezvous.pairing.latency");
|
||||
Assert.All(measurements.SelectMany(static item => item.Tags), static tag =>
|
||||
Assert.Contains(tag.Key, AllowedTagKeys));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MetricScrapeExpiresIdleStateAndReportsExpiryChurn()
|
||||
{
|
||||
EphemeralStateFixture fixture = new();
|
||||
Assert.True(fixture.Store.CreateListing(fixture.ListingCommand()).Succeeded);
|
||||
using RendezvousTelemetry telemetry = new(fixture.Store);
|
||||
List<Measurement> measurements = [];
|
||||
using MeterListener listener = new();
|
||||
listener.InstrumentPublished = (instrument, meterListener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
|
||||
{
|
||||
meterListener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
||||
measurements.Add(new(instrument.Name, value, Tags(tags))));
|
||||
listener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
|
||||
measurements.Add(new(instrument.Name, value, Tags(tags))));
|
||||
listener.Start();
|
||||
|
||||
listener.RecordObservableInstruments();
|
||||
Assert.Equal(
|
||||
1,
|
||||
Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value);
|
||||
|
||||
fixture.Clock.Advance(TimeSpan.FromSeconds(61));
|
||||
measurements.Clear();
|
||||
listener.RecordObservableInstruments();
|
||||
Assert.Equal(
|
||||
0,
|
||||
Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value);
|
||||
Assert.True(Assert.Single(
|
||||
measurements,
|
||||
static item => item.Name == "rendezvous.store.expiry_churn").Value >= 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AuditTrailFingerprintsIdentifiersEnforcesRetentionAndBoundsCapacity()
|
||||
{
|
||||
EphemeralStateFixture fixture = new();
|
||||
using RendezvousTelemetry telemetry = new(fixture.Store);
|
||||
CapturingLogger<AuditTrail> logger = new();
|
||||
ManualTimeProvider time = new(new DateTimeOffset(2026, 7, 16, 0, 0, 0, TimeSpan.Zero));
|
||||
AuditTrail audit = new(
|
||||
Options.Create(new AuditOptions { MaxEntries = 100, RetentionDays = 1 }),
|
||||
logger,
|
||||
telemetry,
|
||||
time);
|
||||
const string actor = "operator-secret-subject";
|
||||
const string target = "player-secret-subject";
|
||||
|
||||
for (int index = 0; index < 101; index++)
|
||||
{
|
||||
audit.Record(actor, "revoke-principal", "succeeded", "principal", target, "safe-correlation");
|
||||
}
|
||||
|
||||
IReadOnlyList<AuditEntry> bounded = audit.GetEntriesForTests();
|
||||
Assert.Equal(100, bounded.Count);
|
||||
Assert.All(bounded, entry =>
|
||||
{
|
||||
Assert.DoesNotContain(actor, entry.ToString(), StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(target, entry.ToString(), StringComparison.Ordinal);
|
||||
Assert.NotEqual(actor, entry.ActorFingerprint);
|
||||
Assert.NotEqual(target, entry.TargetFingerprint);
|
||||
});
|
||||
Assert.DoesNotContain(actor, string.Join('|', logger.Messages), StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(target, string.Join('|', logger.Messages), StringComparison.Ordinal);
|
||||
|
||||
time.Advance(TimeSpan.FromDays(2));
|
||||
Assert.Empty(audit.GetEntriesForTests());
|
||||
Assert.Empty(audit.GetAggregateCounts());
|
||||
audit.Record(actor, "inspect-status", "succeeded", "service", "rendezvous", "safe-correlation");
|
||||
Assert.Single(audit.GetEntriesForTests());
|
||||
Assert.Equal(1, audit.GetAggregateCounts()["inspect-status:succeeded"]);
|
||||
}
|
||||
|
||||
private static KeyValuePair<string, object?>[] Tags(
|
||||
ReadOnlySpan<KeyValuePair<string, object?>> tags) => tags.ToArray();
|
||||
|
||||
private sealed record Measurement(
|
||||
string Name,
|
||||
double Value,
|
||||
KeyValuePair<string, object?>[] Tags);
|
||||
|
||||
private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider
|
||||
{
|
||||
private DateTimeOffset _now = now;
|
||||
public override DateTimeOffset GetUtcNow() => _now;
|
||||
public void Advance(TimeSpan duration) => _now += duration;
|
||||
}
|
||||
|
||||
private sealed class NoopIntroductionSink : INatIntroductionSink
|
||||
{
|
||||
public static NoopIntroductionSink Instance { get; } = new();
|
||||
public void Introduce(NatIntroductionPlan plan)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class CapturingLogger<T> : ILogger<T>
|
||||
{
|
||||
public List<string> Messages { get; } = [];
|
||||
|
||||
public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel,
|
||||
EventId eventId,
|
||||
TState state,
|
||||
Exception? exception,
|
||||
Func<TState, Exception?, string> formatter) => Messages.Add(formatter(state, exception));
|
||||
}
|
||||
|
||||
internal sealed class CapturingLoggerProvider : ILoggerProvider
|
||||
{
|
||||
public ConcurrentQueue<string> Messages { get; } = new();
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new Sink(Messages);
|
||||
|
||||
public void Dispose() => GC.SuppressFinalize(this);
|
||||
|
||||
private sealed class Sink(ConcurrentQueue<string> messages) : ILogger
|
||||
{
|
||||
public IDisposable BeginScope<TState>(TState state) where TState : notnull => Scope.Instance;
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(
|
||||
LogLevel logLevel,
|
||||
EventId eventId,
|
||||
TState state,
|
||||
Exception? exception,
|
||||
Func<TState, Exception?, string> formatter) => messages.Enqueue(formatter(state, exception));
|
||||
}
|
||||
|
||||
private sealed class Scope : IDisposable
|
||||
{
|
||||
public static Scope Instance { get; } = new();
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Net;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Net.Http.Json;
|
||||
using System.Text.Json;
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
using FinalFactory.Rendezvous.Server.Abuse;
|
||||
using FinalFactory.Rendezvous.Server.Http;
|
||||
using FinalFactory.Rendezvous.Server.JoinAttempts;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
using FinalFactory.Rendezvous.Server.Operations;
|
||||
using FinalFactory.Rendezvous.Server.Provisioning;
|
||||
using FinalFactory.Rendezvous.Server.Sessions;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
using FinalFactory.Rendezvous.Server.Transport;
|
||||
using FinalFactory.Rendezvous.Tests.Observability;
|
||||
using FinalFactory.Rendezvous.Tests.Provisioning;
|
||||
using FinalFactory.Rendezvous.Tests.State;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Tests.Operations;
|
||||
|
||||
[Collection(RendezvousTelemetryIsolation.Name)]
|
||||
public sealed class OperatorEndpointTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task OperatorSurfaceSeparatesAuthenticationConfirmsActionsAndRedactsInspection()
|
||||
{
|
||||
await using OperatorTestHost host = await OperatorTestHost.StartAsync();
|
||||
List<string> telemetryData = [];
|
||||
using MeterListener meterListener = new();
|
||||
meterListener.InstrumentPublished = (instrument, listener) =>
|
||||
{
|
||||
if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
|
||||
{
|
||||
listener.EnableMeasurementEvents(instrument);
|
||||
}
|
||||
};
|
||||
meterListener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
|
||||
CaptureMeasurement(telemetryData, instrument, value, tags));
|
||||
meterListener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
|
||||
CaptureMeasurement(telemetryData, instrument, value, tags));
|
||||
meterListener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
|
||||
CaptureMeasurement(telemetryData, instrument, value, tags));
|
||||
meterListener.Start();
|
||||
using ActivityListener activityListener = new()
|
||||
{
|
||||
ShouldListenTo = static source => source.Name == RendezvousTelemetry.ActivitySourceName,
|
||||
Sample = static (ref ActivityCreationOptions<ActivityContext> _) =>
|
||||
ActivitySamplingResult.AllData,
|
||||
ActivityStopped = activity =>
|
||||
{
|
||||
telemetryData.Add(activity.DisplayName);
|
||||
telemetryData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}"));
|
||||
},
|
||||
};
|
||||
ActivitySource.AddActivityListener(activityListener);
|
||||
|
||||
using HttpResponseMessage liveBeforeDependencies = await host.Client.GetAsync("/health/live");
|
||||
Assert.Equal(HttpStatusCode.OK, liveBeforeDependencies.StatusCode);
|
||||
using HttpResponseMessage readyBeforeUdp = await host.Client.GetAsync("/health/ready");
|
||||
Assert.Equal(HttpStatusCode.ServiceUnavailable, readyBeforeUdp.StatusCode);
|
||||
|
||||
using HttpResponseMessage unauthenticated = await host.Client.GetAsync("/v1/operator/status");
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, unauthenticated.StatusCode);
|
||||
AuthenticationHeaderValue challenge = Assert.Single(
|
||||
unauthenticated.Headers.WwwAuthenticate);
|
||||
Assert.Equal("Bearer", challenge.Scheme);
|
||||
Assert.Equal("realm=\"operator\"", challenge.Parameter);
|
||||
|
||||
using HttpResponseMessage publisher = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Get,
|
||||
"/v1/operator/status",
|
||||
host.PublisherCredential);
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, publisher.StatusCode);
|
||||
|
||||
using HttpResponseMessage status = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Get,
|
||||
"/v1/operator/status",
|
||||
host.ReadOnlyOperatorCredential);
|
||||
Assert.Equal(HttpStatusCode.OK, status.StatusCode);
|
||||
string statusJson = await status.Content.ReadAsStringAsync();
|
||||
Assert.Contains("not-ready", statusJson, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(host.OwnerCanary, statusJson, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("203.0.113.25", statusJson, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("metadata", statusJson, StringComparison.OrdinalIgnoreCase);
|
||||
OperatorStatusResponse? operatorStatus = JsonSerializer.Deserialize<OperatorStatusResponse>(
|
||||
statusJson,
|
||||
ContractJson.Options);
|
||||
Assert.Contains(operatorStatus!.Tenants, static tenant =>
|
||||
tenant.GameId == "space-game"
|
||||
&& tenant.EnvironmentId == "production"
|
||||
&& tenant.Status == "enabled");
|
||||
Assert.Contains(operatorStatus.SigningKeys, static key =>
|
||||
key.KeyId == OperatorTestHost.OperatorKeyId
|
||||
&& key.Status == "signing"
|
||||
&& key.CredentialKinds.SequenceEqual(["Operator"]));
|
||||
|
||||
await host.StartUdpAsync();
|
||||
using HttpResponseMessage readyAfterUdp = await host.Client.GetAsync("/health/ready");
|
||||
Assert.Equal(HttpStatusCode.OK, readyAfterUdp.StatusCode);
|
||||
|
||||
using HttpResponseMessage exception = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/test/exception",
|
||||
host.FullOperatorCredential);
|
||||
Assert.Equal(HttpStatusCode.InternalServerError, exception.StatusCode);
|
||||
using HttpResponseMessage saturatedPublic = await host.Client.GetAsync("/test/public");
|
||||
Assert.Equal(HttpStatusCode.TooManyRequests, saturatedPublic.StatusCode);
|
||||
|
||||
using HttpResponseMessage forbidden = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/drain",
|
||||
host.ReadOnlyOperatorCredential,
|
||||
new BeginDrainRequest { Confirmation = "DRAIN" });
|
||||
Assert.Equal(HttpStatusCode.Forbidden, forbidden.StatusCode);
|
||||
|
||||
SessionListingId listingId = host.CreateListing(host.OwnerCanary);
|
||||
using HttpResponseMessage unconfirmedListing = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/listings/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokeListingRequest
|
||||
{
|
||||
ListingId = listingId.ToString(),
|
||||
ConfirmListingId = Guid.NewGuid().ToString("D"),
|
||||
});
|
||||
Assert.Equal(HttpStatusCode.BadRequest, unconfirmedListing.StatusCode);
|
||||
Assert.True(host.Store.GetListing(listingId, false).Succeeded);
|
||||
|
||||
using HttpResponseMessage revokedListing = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/listings/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokeListingRequest
|
||||
{
|
||||
ListingId = listingId.ToString(),
|
||||
ConfirmListingId = listingId.ToString(),
|
||||
});
|
||||
Assert.Equal(HttpStatusCode.OK, revokedListing.StatusCode);
|
||||
Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(listingId, false).Code);
|
||||
|
||||
const string principalCanary = "publisher-player-canary";
|
||||
SessionListingId principalListing = host.CreateListing(principalCanary);
|
||||
using HttpResponseMessage unconfirmedPrincipal = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/principals/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokePrincipalRequest
|
||||
{
|
||||
Subject = principalCanary,
|
||||
ConfirmSubject = "different-subject",
|
||||
LifetimeSeconds = 60,
|
||||
});
|
||||
Assert.Equal(HttpStatusCode.BadRequest, unconfirmedPrincipal.StatusCode);
|
||||
Assert.True(host.Store.GetListing(principalListing, false).Succeeded);
|
||||
|
||||
using HttpResponseMessage revokedPrincipal = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/principals/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokePrincipalRequest
|
||||
{
|
||||
Subject = principalCanary,
|
||||
ConfirmSubject = principalCanary,
|
||||
LifetimeSeconds = 60,
|
||||
});
|
||||
Assert.Equal(HttpStatusCode.OK, revokedPrincipal.StatusCode);
|
||||
OperatorActionResponse? principalResult = await revokedPrincipal.Content
|
||||
.ReadFromJsonAsync<OperatorActionResponse>(ContractJson.Options);
|
||||
Assert.Equal(1, principalResult!.AffectedResources);
|
||||
Assert.Equal(StoreResultCode.NotFound, host.Store.GetListing(principalListing, false).Code);
|
||||
StoreResult<StoredListing> blockedPublisher = host.CreateListingResult(
|
||||
principalCanary,
|
||||
out _);
|
||||
Assert.Equal(StoreResultCode.Revoked, blockedPublisher.Code);
|
||||
|
||||
using HttpResponseMessage drain = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/drain",
|
||||
host.FullOperatorCredential,
|
||||
new BeginDrainRequest { Confirmation = "DRAIN" });
|
||||
Assert.Equal(HttpStatusCode.OK, drain.StatusCode);
|
||||
Assert.True(host.Store.IsDraining);
|
||||
using HttpResponseMessage liveDuringDrain = await host.Client.GetAsync("/health/live");
|
||||
Assert.Equal(HttpStatusCode.OK, liveDuringDrain.StatusCode);
|
||||
using HttpResponseMessage readyDuringDrain = await host.Client.GetAsync("/health/ready");
|
||||
Assert.Equal(HttpStatusCode.ServiceUnavailable, readyDuringDrain.StatusCode);
|
||||
|
||||
using HttpResponseMessage firstPublisherKeyRevocation = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/keys/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" });
|
||||
Assert.Equal(HttpStatusCode.OK, firstPublisherKeyRevocation.StatusCode);
|
||||
using HttpResponseMessage repeatedPublisherKeyRevocation = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/keys/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokeSigningKeyRequest { KeyId = "key-1", ConfirmKeyId = "key-1" });
|
||||
Assert.Equal(HttpStatusCode.OK, repeatedPublisherKeyRevocation.StatusCode);
|
||||
|
||||
using HttpResponseMessage unconfirmedKey = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/keys/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokeSigningKeyRequest
|
||||
{
|
||||
KeyId = OperatorTestHost.OperatorKeyId,
|
||||
ConfirmKeyId = "different-key",
|
||||
});
|
||||
Assert.Equal(HttpStatusCode.BadRequest, unconfirmedKey.StatusCode);
|
||||
|
||||
using HttpResponseMessage revokedKey = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Post,
|
||||
"/v1/operator/keys/revoke",
|
||||
host.FullOperatorCredential,
|
||||
new RevokeSigningKeyRequest
|
||||
{
|
||||
KeyId = OperatorTestHost.OperatorKeyId,
|
||||
ConfirmKeyId = OperatorTestHost.OperatorKeyId,
|
||||
});
|
||||
Assert.Equal(HttpStatusCode.OK, revokedKey.StatusCode);
|
||||
using HttpResponseMessage afterKeyRevocation = await SendAsync(
|
||||
host,
|
||||
HttpMethod.Get,
|
||||
"/v1/operator/status",
|
||||
host.FullOperatorCredential);
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, afterKeyRevocation.StatusCode);
|
||||
|
||||
string auditText = string.Join('|', host.Audit.GetEntriesForTests());
|
||||
string logText = string.Join('|', host.AuditLogger.Messages.Concat(host.AllLogs.Messages));
|
||||
Assert.DoesNotContain(host.OwnerCanary, auditText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(principalCanary, auditText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(listingId.ToString(), auditText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(host.OwnerCanary, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(principalCanary, logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("exception-secret-canary", logText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(host.FullOperatorCredential, logText, StringComparison.Ordinal);
|
||||
meterListener.RecordObservableInstruments();
|
||||
string telemetryText = string.Join('|', telemetryData);
|
||||
Assert.DoesNotContain(host.OwnerCanary, telemetryText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(principalCanary, telemetryText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("exception-secret-canary", telemetryText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(host.FullOperatorCredential, telemetryText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain(listingId.ToString(), telemetryText, StringComparison.Ordinal);
|
||||
Assert.DoesNotContain("203.0.113.25", telemetryText, StringComparison.Ordinal);
|
||||
Assert.Contains(host.Audit.GetEntriesForTests(), static entry =>
|
||||
entry.Action == "begin-drain" && entry.Result == "succeeded");
|
||||
Assert.Contains(host.Audit.GetEntriesForTests(), static entry =>
|
||||
entry.Action == "revoke-listing" && entry.Result == "rejected");
|
||||
Assert.Contains(host.Audit.GetEntriesForTests(), static entry =>
|
||||
entry.Action == "begin-drain" && entry.Result == "forbidden");
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseMessage> SendAsync(
|
||||
OperatorTestHost host,
|
||||
HttpMethod method,
|
||||
string path,
|
||||
string bearer,
|
||||
object? body = null)
|
||||
{
|
||||
using HttpRequestMessage request = new(method, path);
|
||||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", bearer);
|
||||
if (body is not null)
|
||||
{
|
||||
request.Content = JsonContent.Create(body, options: ContractJson.Options);
|
||||
}
|
||||
|
||||
return await host.Client.SendAsync(request);
|
||||
}
|
||||
|
||||
private static void CaptureMeasurement<T>(
|
||||
List<string> destination,
|
||||
Instrument instrument,
|
||||
T value,
|
||||
ReadOnlySpan<KeyValuePair<string, object?>> tags)
|
||||
where T : struct
|
||||
{
|
||||
destination.Add($"{instrument.Name}={value}");
|
||||
destination.AddRange(tags.ToArray().Select(static tag => $"{tag.Key}={tag.Value}"));
|
||||
}
|
||||
|
||||
private sealed class OperatorTestHost : IAsyncDisposable
|
||||
{
|
||||
private readonly WebApplication _application;
|
||||
private int _listingSequence;
|
||||
private bool _udpStarted;
|
||||
|
||||
private OperatorTestHost(
|
||||
WebApplication application,
|
||||
HttpClient client,
|
||||
ManualRendezvousClock clock,
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
AuditTrail audit,
|
||||
CapturingLogger<AuditTrail> auditLogger,
|
||||
CapturingLoggerProvider allLogs,
|
||||
string publisherCredential,
|
||||
string readOnlyOperatorCredential,
|
||||
string fullOperatorCredential)
|
||||
{
|
||||
_application = application;
|
||||
Client = client;
|
||||
Clock = clock;
|
||||
Store = store;
|
||||
Audit = audit;
|
||||
AuditLogger = auditLogger;
|
||||
AllLogs = allLogs;
|
||||
PublisherCredential = publisherCredential;
|
||||
ReadOnlyOperatorCredential = readOnlyOperatorCredential;
|
||||
FullOperatorCredential = fullOperatorCredential;
|
||||
}
|
||||
|
||||
internal const string OperatorKeyId = "operator-key";
|
||||
internal string OwnerCanary { get; } = "publisher-owner-canary";
|
||||
internal HttpClient Client { get; }
|
||||
internal ManualRendezvousClock Clock { get; }
|
||||
internal InMemoryEphemeralRendezvousStore Store { get; }
|
||||
internal AuditTrail Audit { get; }
|
||||
internal CapturingLogger<AuditTrail> AuditLogger { get; }
|
||||
internal CapturingLoggerProvider AllLogs { get; }
|
||||
internal string PublisherCredential { get; }
|
||||
internal string ReadOnlyOperatorCredential { get; }
|
||||
internal string FullOperatorCredential { get; }
|
||||
|
||||
internal static async Task<OperatorTestHost> StartAsync()
|
||||
{
|
||||
ManualRendezvousClock clock = new(ProvisioningTestData.Now);
|
||||
EphemeralStoreOptions stateOptions = new();
|
||||
InMemoryEphemeralRendezvousStore store = new(stateOptions, clock, clock);
|
||||
SigningKeyOptions publisherKey = ProvisioningTestData.CreateKey();
|
||||
SigningKeyOptions operatorKey = ProvisioningTestData.CreateKey(
|
||||
OperatorKeyId,
|
||||
"operator-secret",
|
||||
credentialKinds: [PrincipalCredentialKind.Operator],
|
||||
gameId: null,
|
||||
environmentId: null);
|
||||
ProvisioningOptions options = ProvisioningTestData.CreateOptions();
|
||||
options.SigningKeys = [publisherKey, operatorKey];
|
||||
ProvisioningRuntime provisioning = ProvisioningRuntime.Create(
|
||||
options,
|
||||
ProvisioningTestData.CreateSecrets("secret-1", "operator-secret"),
|
||||
clock.UtcNow);
|
||||
string publisherCredential = provisioning.Credentials.Issue(
|
||||
ProvisioningTestData.CreateDedicatedPublisher(),
|
||||
clock.UtcNow);
|
||||
string readOnlyCredential = provisioning.Credentials.Issue(
|
||||
new OperatorPrincipal(
|
||||
"operator-readonly",
|
||||
clock.UtcNow.AddMinutes(10),
|
||||
[OperatorPermission.ReadPolicy]),
|
||||
clock.UtcNow);
|
||||
string fullCredential = provisioning.Credentials.Issue(
|
||||
new OperatorPrincipal(
|
||||
"operator-full",
|
||||
clock.UtcNow.AddMinutes(10),
|
||||
Enum.GetValues<OperatorPermission>()),
|
||||
clock.UtcNow);
|
||||
CapturingLogger<AuditTrail> auditLogger = new();
|
||||
CapturingLoggerProvider allLogs = new();
|
||||
EphemeralCapabilityIssuer capabilities = new();
|
||||
|
||||
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
||||
builder.WebHost.UseUrls("http://127.0.0.1:0");
|
||||
builder.Logging.ClearProviders();
|
||||
builder.Logging.SetMinimumLevel(LogLevel.Debug);
|
||||
builder.Logging.AddProvider(allLogs);
|
||||
builder.Logging.AddFilter(
|
||||
"Microsoft.AspNetCore.Diagnostics.ExceptionHandlerMiddleware",
|
||||
LogLevel.None);
|
||||
builder.Services.ConfigureHttpJsonOptions(static json =>
|
||||
ContractJson.Configure(json.SerializerOptions));
|
||||
builder.Services.Configure<RouteHandlerOptions>(static route =>
|
||||
route.ThrowOnBadRequest = true);
|
||||
builder.Services.AddProblemDetails();
|
||||
builder.Services.AddExceptionHandler<RendezvousExceptionHandler>();
|
||||
builder.Services.AddOptions<AbuseProtectionOptions>().Configure(static abuse =>
|
||||
{
|
||||
abuse.OperatorAllowedAddresses = ["127.0.0.1"];
|
||||
abuse.HttpGlobalRequestsPerWindow = 1;
|
||||
abuse.HttpOptionalRequestsPerWindow = 1;
|
||||
abuse.HttpIpPrefixRequestsPerWindow = 1;
|
||||
abuse.HttpOptionalIpPrefixRequestsPerWindow = 1;
|
||||
});
|
||||
builder.Services.AddOptions<AuditOptions>();
|
||||
builder.Services.AddOptions<UdpMediatorOptions>().Configure(static udp =>
|
||||
{
|
||||
udp.ListenAddress = "127.0.0.1";
|
||||
udp.Port = 0;
|
||||
});
|
||||
builder.Services.AddSingleton(provisioning);
|
||||
builder.Services.AddSingleton(provisioning.Policies);
|
||||
builder.Services.AddSingleton(provisioning.Credentials);
|
||||
builder.Services.AddSingleton(provisioning.PublisherAuthorization);
|
||||
builder.Services.AddSingleton(store);
|
||||
builder.Services.AddSingleton<IEphemeralRendezvousStore>(store);
|
||||
builder.Services.AddSingleton<IWallClock>(clock);
|
||||
builder.Services.AddSingleton<IMonotonicClock>(clock);
|
||||
builder.Services.AddSingleton(capabilities);
|
||||
builder.Services.AddSingleton<ISessionCapabilityService>(capabilities);
|
||||
builder.Services.AddSingleton<JoinAttemptCursorCodec>();
|
||||
builder.Services.AddSingleton<JoinAttemptService>();
|
||||
builder.Services.AddSingleton<RendezvousTelemetry>();
|
||||
builder.Services.AddSingleton<AbuseProtectionService>();
|
||||
builder.Services.AddSingleton<NatMediationProcessor>();
|
||||
builder.Services.AddSingleton<UdpMediatorService>();
|
||||
builder.Services.AddSingleton(new ProvisioningReadiness(true));
|
||||
builder.Services.AddSingleton<RendezvousReadiness>();
|
||||
builder.Services.AddSingleton<ILogger<AuditTrail>>(auditLogger);
|
||||
builder.Services.AddSingleton<AuditTrail>();
|
||||
builder.Services.AddSingleton<OperatorService>();
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
app.UseMiddleware<TelemetryMiddleware>();
|
||||
app.UseExceptionHandler();
|
||||
app.UseMiddleware<HttpAbuseProtectionMiddleware>();
|
||||
app.MapOperatorEndpoints();
|
||||
app.MapRendezvousHealthEndpoints();
|
||||
app.MapGet("/test/public", static () => Results.Ok()).WithName("TestPublic");
|
||||
app.MapPost(
|
||||
"/test/exception",
|
||||
static IResult () => throw new InvalidOperationException("exception-secret-canary"))
|
||||
.WithName("TestSecretException");
|
||||
await app.StartAsync();
|
||||
IServer server = app.Services.GetRequiredService<IServer>();
|
||||
string address = Assert.Single(server.Features.Get<IServerAddressesFeature>()!.Addresses);
|
||||
return new OperatorTestHost(
|
||||
app,
|
||||
new HttpClient { BaseAddress = new Uri(address) },
|
||||
clock,
|
||||
store,
|
||||
app.Services.GetRequiredService<AuditTrail>(),
|
||||
auditLogger,
|
||||
allLogs,
|
||||
publisherCredential,
|
||||
readOnlyCredential,
|
||||
fullCredential);
|
||||
}
|
||||
|
||||
internal SessionListingId CreateListing(string owner)
|
||||
{
|
||||
StoreResult<StoredListing> result = CreateListingResult(owner, out SessionListingId listingId);
|
||||
Assert.True(result.Succeeded);
|
||||
return listingId;
|
||||
}
|
||||
|
||||
internal StoreResult<StoredListing> CreateListingResult(
|
||||
string owner,
|
||||
out SessionListingId listingId)
|
||||
{
|
||||
int sequence = Interlocked.Increment(ref _listingSequence);
|
||||
listingId = new(Guid.NewGuid());
|
||||
StoreResult<StoredListing> result = Store.CreateListing(new(
|
||||
$"operator-listing-{sequence}",
|
||||
$"operator-request-{sequence}",
|
||||
new ListingDefinition
|
||||
{
|
||||
ListingId = listingId,
|
||||
LeaseId = new(Guid.NewGuid()),
|
||||
Scope = new(new GameId("space-game"), new EnvironmentId("production")),
|
||||
OwnerSubject = owner,
|
||||
RegionId = new("eu-central"),
|
||||
ProtocolVersion = 7,
|
||||
BuildVersion = "1.0.0",
|
||||
DisplayName = "Operator test listing",
|
||||
Visibility = ListingVisibility.Public,
|
||||
TrustMode = PublisherTrustMode.ManagedDedicated,
|
||||
CurrentPlayers = 1,
|
||||
MaximumPlayers = 4,
|
||||
Metadata = new Dictionary<string, string> { ["mode"] = "online-coop" },
|
||||
LeaseFingerprint = new("lease-fingerprint"),
|
||||
HostPresenceHandle = new(Guid.NewGuid()),
|
||||
HostPresenceFingerprint = new("presence-fingerprint"),
|
||||
CapabilityDerivationSalt = new string('A', 43),
|
||||
}));
|
||||
return result;
|
||||
}
|
||||
|
||||
internal async Task StartUdpAsync()
|
||||
{
|
||||
await _application.Services.GetRequiredService<UdpMediatorService>()
|
||||
.StartAsync(CancellationToken.None);
|
||||
_udpStarted = true;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
Client.Dispose();
|
||||
if (_udpStarted)
|
||||
{
|
||||
await _application.Services.GetRequiredService<UdpMediatorService>()
|
||||
.StopAsync(CancellationToken.None);
|
||||
}
|
||||
await _application.StopAsync();
|
||||
await _application.DisposeAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,7 @@ public sealed class PrincipalCredentialTests
|
||||
CredentialValidationError.SignatureInvalid,
|
||||
service.Validate(tampered, ProvisioningTestData.Now).Error);
|
||||
Assert.True(keys.Revoke("key-1"));
|
||||
Assert.True(keys.Revoke("key-1"));
|
||||
Assert.Equal(
|
||||
CredentialValidationError.KeyRevoked,
|
||||
service.Validate(token, ProvisioningTestData.Now).Error);
|
||||
|
||||
@@ -6,6 +6,7 @@ using FinalFactory.Rendezvous.Server.Http;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.HttpOverrides;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
@@ -86,6 +87,9 @@ public sealed class AbuseProtectionTests
|
||||
AbuseProtectionService protection = new(Options.Create(options));
|
||||
IPAddress source = IPAddress.Parse("198.51.100.10");
|
||||
|
||||
Assert.True(protection.IsOperatorSourceAllowed(source));
|
||||
Assert.True(protection.IsOperatorSourceAllowed(IPAddress.Parse("::ffff:198.51.100.10")));
|
||||
Assert.False(protection.IsOperatorSourceAllowed(IPAddress.Parse("198.51.100.11")));
|
||||
AssertAccepted(protection, source, "BrowseSessions");
|
||||
AssertAccepted(protection, source, "BrowseSessions");
|
||||
AssertRejected(protection, source, "BrowseSessions");
|
||||
@@ -93,6 +97,57 @@ public sealed class AbuseProtectionTests
|
||||
AssertRejected(protection, source, "RenewSessionLease");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PublicSaturationCannotConsumeTheOperatorPartition()
|
||||
{
|
||||
AbuseProtectionOptions options = PermissiveOptions();
|
||||
options.HttpGlobalRequestsPerWindow = 1;
|
||||
options.HttpOptionalRequestsPerWindow = 1;
|
||||
options.OperatorGlobalRequestsPerWindow = 1;
|
||||
AbuseProtectionService protection = new(Options.Create(options));
|
||||
IPAddress source = IPAddress.Parse("198.51.100.10");
|
||||
|
||||
AssertAccepted(protection, source, "BrowseSessions");
|
||||
AssertRejected(protection, source, "BrowseSessions");
|
||||
Assert.True(protection.TryAcquireOperatorIngress(source, out var lease, out _));
|
||||
lease!.Dispose();
|
||||
Assert.False(protection.TryAcquireOperatorIngress(source, out _, out _));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeniedOperatorSourcesConsumeTheBoundedPublicPartition()
|
||||
{
|
||||
AbuseProtectionOptions options = PermissiveOptions();
|
||||
options.OperatorAllowedAddresses = ["192.0.2.10"];
|
||||
options.HttpGlobalRequestsPerWindow = 1;
|
||||
options.HttpOptionalRequestsPerWindow = 1;
|
||||
options.HttpIpPrefixRequestsPerWindow = 1;
|
||||
options.HttpOptionalIpPrefixRequestsPerWindow = 1;
|
||||
AbuseProtectionService protection = new(Options.Create(options));
|
||||
bool dispatched = false;
|
||||
HttpAbuseProtectionMiddleware middleware = new(
|
||||
_ =>
|
||||
{
|
||||
dispatched = true;
|
||||
return Task.CompletedTask;
|
||||
},
|
||||
protection);
|
||||
|
||||
DefaultHttpContext first = Context("198.51.100.10");
|
||||
first.SetEndpoint(new Endpoint(
|
||||
_ => Task.CompletedTask,
|
||||
new EndpointMetadataCollection(new EndpointNameMetadata("GetOperatorStatus")),
|
||||
"operator-status"));
|
||||
await middleware.InvokeAsync(first);
|
||||
Assert.Equal(StatusCodes.Status404NotFound, first.Response.StatusCode);
|
||||
|
||||
DefaultHttpContext repeated = Context("198.51.100.10");
|
||||
repeated.SetEndpoint(first.GetEndpoint());
|
||||
await middleware.InvokeAsync(repeated);
|
||||
Assert.Equal(StatusCodes.Status429TooManyRequests, repeated.Response.StatusCode);
|
||||
Assert.False(dispatched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResourceBudgetsRemainIsolatedAcrossTenantAndPrincipalScopes()
|
||||
{
|
||||
@@ -354,7 +409,8 @@ public sealed class AbuseProtectionTests
|
||||
{
|
||||
const string canary = "credential-canary <script> endpoint=203.0.113.8:9000";
|
||||
DefaultHttpContext context = Context("198.51.100.10");
|
||||
RendezvousExceptionHandler handler = new();
|
||||
RendezvousExceptionHandler handler = new(
|
||||
NullLogger<RendezvousExceptionHandler>.Instance);
|
||||
|
||||
Assert.True(await handler.TryHandleAsync(
|
||||
context,
|
||||
@@ -438,6 +494,11 @@ public sealed class AbuseProtectionTests
|
||||
HealthGlobalConcurrency = 10_000,
|
||||
HealthIpPrefixRequestsPerWindow = 10_000,
|
||||
HealthIpPrefixConcurrency = 1_000,
|
||||
OperatorAllowedAddresses = ["198.51.100.10"],
|
||||
OperatorGlobalRequestsPerWindow = 10_000,
|
||||
OperatorGlobalConcurrency = 10_000,
|
||||
OperatorIpPrefixRequestsPerWindow = 10_000,
|
||||
OperatorIpPrefixConcurrency = 1_000,
|
||||
HttpGlobalRequestsPerWindow = 10_000,
|
||||
HttpOptionalRequestsPerWindow = 9_000,
|
||||
HttpIpPrefixRequestsPerWindow = 10_000,
|
||||
|
||||
@@ -436,7 +436,7 @@ public sealed class InMemoryEphemeralRendezvousStoreTests
|
||||
StoreResult<int> revoked = fixture.Store.RevokePrincipal(command.Listing.OwnerSubject, TimeSpan.FromMinutes(1));
|
||||
|
||||
Assert.True(revoked.Succeeded);
|
||||
Assert.Equal(2, revoked.Value);
|
||||
Assert.Equal(4, revoked.Value);
|
||||
Assert.Equal(StoreResultCode.NotFound, fixture.Store.GetListing(listing.Definition.ListingId, false).Code);
|
||||
Assert.Equal(StoreResultCode.NotFound, fixture.Store.BindAttemptEndpoint(new(
|
||||
attempt.MediationHandle,
|
||||
|
||||
Reference in New Issue
Block a user