using System.Collections.Concurrent; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Net; using FinalFactory.Rendezvous.Server.Observability; using FinalFactory.Rendezvous.Server.State; using FinalFactory.Rendezvous.Server.Transport; using FinalFactory.Rendezvous.Tests.State; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace FinalFactory.Rendezvous.Tests.Observability; [CollectionDefinition(RendezvousTelemetryIsolation.Name, DisableParallelization = true)] public sealed class RendezvousTelemetryIsolation { public const string Name = "Rendezvous telemetry"; } [Collection(RendezvousTelemetryIsolation.Name)] public sealed class ObservabilityTests { private static readonly HashSet AllowedTagKeys = [ "operation", "status_code", "result", "transport", "partition", "action", "outcome", "elapsed_bucket", ]; [Fact] public void MetricsAndTracesUseBoundedDimensionsWithoutSensitiveValues() { EphemeralStateFixture fixture = new(); using RendezvousTelemetry telemetry = new(fixture.Store); List measurements = []; using MeterListener meterListener = new(); meterListener.InstrumentPublished = (instrument, listener) => { if (instrument.Meter.Name == RendezvousTelemetry.MeterName) { listener.EnableMeasurementEvents(instrument); } }; meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => measurements.Add(new(instrument.Name, value, Tags(tags)))); meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => measurements.Add(new(instrument.Name, value, Tags(tags)))); meterListener.SetMeasurementEventCallback((instrument, value, tags, _) => measurements.Add(new(instrument.Name, value, Tags(tags)))); meterListener.Start(); Activity? observed = null; List activityData = []; using ActivityListener activityListener = new() { ShouldListenTo = source => source.Name == RendezvousTelemetry.ActivitySourceName, Sample = static (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, ActivityStopped = activity => { observed = activity; activityData.Add(activity.DisplayName); activityData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}")); }, }; ActivitySource.AddActivityListener(activityListener); const string secret = "secret-player-token-canary"; using (telemetry.StartActivity("HTTP GetOperatorStatus", ActivityKind.Server)) { telemetry.RecordHttp("GetOperatorStatus", 200, 3.5); telemetry.RecordUdp("frozen", "Introduced", 1.25); telemetry.RecordLimiterDrop("udp", "rate-or-concurrency"); telemetry.RecordAudit("revoke-listing", "succeeded"); telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond"); telemetry.RecordOperatorAuthentication("accepted"); telemetry.RecordPairingLatency(12.5); } NatMediationProcessor processor = new(null!, null!, null!, telemetry: telemetry); Assert.Equal( NatMediationResult.Dropped, processor.ProcessRequest( new IPEndPoint(IPAddress.Parse("10.0.0.8"), 9000), new IPEndPoint(IPAddress.Parse("203.0.113.8"), 50000), secret, NoopIntroductionSink.Instance)); meterListener.RecordObservableInstruments(); Assert.NotNull(observed); string flattened = string.Join('|', measurements.Select(static item => item.ToString())); Assert.DoesNotContain(secret, flattened, StringComparison.Ordinal); Assert.DoesNotContain(secret, string.Join('|', activityData), StringComparison.Ordinal); Assert.Contains(measurements, static item => item.Name == "rendezvous.http.requests"); Assert.Contains(measurements, static item => item.Name == "rendezvous.udp.results"); Assert.Contains(measurements, static item => item.Name == "rendezvous.limiter.drops"); Assert.Contains(measurements, static item => item.Name == "rendezvous.queue.depth"); Assert.Contains(measurements, static item => item.Name == "rendezvous.store.active_leases"); Assert.Contains(measurements, static item => item.Name == "rendezvous.store.expiry_churn"); Assert.Contains(measurements, static item => item.Name == "rendezvous.pairing.latency"); Assert.All(measurements.SelectMany(static item => item.Tags), static tag => Assert.Contains(tag.Key, AllowedTagKeys)); } [Fact] public void MetricScrapeExpiresIdleStateAndReportsExpiryChurn() { EphemeralStateFixture fixture = new(); Assert.True(fixture.Store.CreateListing(fixture.ListingCommand()).Succeeded); using RendezvousTelemetry telemetry = new(fixture.Store); List measurements = []; using MeterListener listener = new(); listener.InstrumentPublished = (instrument, meterListener) => { if (instrument.Meter.Name == RendezvousTelemetry.MeterName) { meterListener.EnableMeasurementEvents(instrument); } }; listener.SetMeasurementEventCallback((instrument, value, tags, _) => measurements.Add(new(instrument.Name, value, Tags(tags)))); listener.SetMeasurementEventCallback((instrument, value, tags, _) => measurements.Add(new(instrument.Name, value, Tags(tags)))); listener.Start(); listener.RecordObservableInstruments(); Assert.Equal( 1, Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value); fixture.Clock.Advance(TimeSpan.FromSeconds(61)); measurements.Clear(); listener.RecordObservableInstruments(); Assert.Equal( 0, Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value); Assert.True(Assert.Single( measurements, static item => item.Name == "rendezvous.store.expiry_churn").Value >= 1); } [Fact] public void AuditTrailFingerprintsIdentifiersEnforcesRetentionAndBoundsCapacity() { EphemeralStateFixture fixture = new(); using RendezvousTelemetry telemetry = new(fixture.Store); CapturingLogger logger = new(); ManualTimeProvider time = new(new DateTimeOffset(2026, 7, 16, 0, 0, 0, TimeSpan.Zero)); AuditTrail audit = new( Options.Create(new AuditOptions { MaxEntries = 100, RetentionDays = 1 }), logger, telemetry, time); const string actor = "operator-secret-subject"; const string target = "player-secret-subject"; for (int index = 0; index < 101; index++) { audit.Record(actor, "revoke-principal", "succeeded", "principal", target, "safe-correlation"); } IReadOnlyList bounded = audit.GetEntriesForTests(); Assert.Equal(100, bounded.Count); Assert.All(bounded, entry => { Assert.DoesNotContain(actor, entry.ToString(), StringComparison.Ordinal); Assert.DoesNotContain(target, entry.ToString(), StringComparison.Ordinal); Assert.NotEqual(actor, entry.ActorFingerprint); Assert.NotEqual(target, entry.TargetFingerprint); }); Assert.DoesNotContain(actor, string.Join('|', logger.Messages), StringComparison.Ordinal); Assert.DoesNotContain(target, string.Join('|', logger.Messages), StringComparison.Ordinal); time.Advance(TimeSpan.FromDays(2)); Assert.Empty(audit.GetEntriesForTests()); Assert.Empty(audit.GetAggregateCounts()); audit.Record(actor, "inspect-status", "succeeded", "service", "rendezvous", "safe-correlation"); Assert.Single(audit.GetEntriesForTests()); Assert.Equal(1, audit.GetAggregateCounts()["inspect-status:succeeded"]); } private static KeyValuePair[] Tags( ReadOnlySpan> tags) => tags.ToArray(); private sealed record Measurement( string Name, double Value, KeyValuePair[] Tags); private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider { private DateTimeOffset _now = now; public override DateTimeOffset GetUtcNow() => _now; public void Advance(TimeSpan duration) => _now += duration; } private sealed class NoopIntroductionSink : INatIntroductionSink { public static NoopIntroductionSink Instance { get; } = new(); public void Introduce(NatIntroductionPlan plan) { } } } internal sealed class CapturingLogger : ILogger { public List Messages { get; } = []; public IDisposable? BeginScope(TState state) where TState : notnull => null; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => Messages.Add(formatter(state, exception)); } internal sealed class CapturingLoggerProvider : ILoggerProvider { public ConcurrentQueue Messages { get; } = new(); public ILogger CreateLogger(string categoryName) => new Sink(Messages); public void Dispose() => GC.SuppressFinalize(this); private sealed class Sink(ConcurrentQueue messages) : ILogger { public IDisposable BeginScope(TState state) where TState : notnull => Scope.Instance; public bool IsEnabled(LogLevel logLevel) => true; public void Log( LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => messages.Enqueue(formatter(state, exception)); } private sealed class Scope : IDisposable { public static Scope Instance { get; } = new(); public void Dispose() { } } }