Rendezvous/tests/FinalFactory.Rendezvous.Tests/Observability/ObservabilityTests.cs

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Net;
using FinalFactory.Rendezvous.Server.Observability;
using FinalFactory.Rendezvous.Server.State;
using FinalFactory.Rendezvous.Server.Transport;
using FinalFactory.Rendezvous.Tests.State;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace FinalFactory.Rendezvous.Tests.Observability;

[CollectionDefinition(RendezvousTelemetryIsolation.Name, DisableParallelization = true)]
public sealed class RendezvousTelemetryIsolation
{
    public const string Name = "Rendezvous telemetry";
}

[Collection(RendezvousTelemetryIsolation.Name)]
public sealed class ObservabilityTests
{
    private static readonly HashSet<string> AllowedTagKeys =
    [
        "operation",
        "status_code",
        "result",
        "transport",
        "partition",
        "action",
        "outcome",
        "elapsed_bucket",
    ];

    [Fact]
    public void MetricsAndTracesUseBoundedDimensionsWithoutSensitiveValues()
    {
        EphemeralStateFixture fixture = new();
        using RendezvousTelemetry telemetry = new(fixture.Store);
        List<Measurement> measurements = [];
        using MeterListener meterListener = new();
        meterListener.InstrumentPublished = (instrument, listener) =>
        {
            if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
            {
                listener.EnableMeasurementEvents(instrument);
            }
        };
        meterListener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
            measurements.Add(new(instrument.Name, value, Tags(tags))));
        meterListener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
            measurements.Add(new(instrument.Name, value, Tags(tags))));
        meterListener.SetMeasurementEventCallback<double>((instrument, value, tags, _) =>
            measurements.Add(new(instrument.Name, value, Tags(tags))));
        meterListener.Start();

        Activity? observed = null;
        List<string> activityData = [];
        using ActivityListener activityListener = new()
        {
            ShouldListenTo = source => source.Name == RendezvousTelemetry.ActivitySourceName,
            Sample = static (ref ActivityCreationOptions<ActivityContext> _) =>
                ActivitySamplingResult.AllData,
            ActivityStopped = activity =>
            {
                observed = activity;
                activityData.Add(activity.DisplayName);
                activityData.AddRange(activity.TagObjects.Select(static tag => $"{tag.Key}={tag.Value}"));
            },
        };
        ActivitySource.AddActivityListener(activityListener);

        const string secret = "secret-player-token-canary";
        using (telemetry.StartActivity("HTTP GetOperatorStatus", ActivityKind.Server))
        {
            telemetry.RecordHttp("GetOperatorStatus", 200, 3.5);
            telemetry.RecordUdp("frozen", "Introduced", 1.25);
            telemetry.RecordLimiterDrop("udp", "rate-or-concurrency");
            telemetry.RecordAudit("revoke-listing", "succeeded");
            telemetry.RecordConnectionOutcome("Connected", "UnderOneSecond");
            telemetry.RecordOperatorAuthentication("accepted");
            telemetry.RecordPairingLatency(12.5);
        }
        NatMediationProcessor processor = new(null!, null!, null!, telemetry: telemetry);
        Assert.Equal(
            NatMediationResult.Dropped,
            processor.ProcessRequest(
                new IPEndPoint(IPAddress.Parse("10.0.0.8"), 9000),
                new IPEndPoint(IPAddress.Parse("203.0.113.8"), 50000),
                secret,
                NoopIntroductionSink.Instance));

        meterListener.RecordObservableInstruments();
        Assert.NotNull(observed);
        string flattened = string.Join('|', measurements.Select(static item => item.ToString()));
        Assert.DoesNotContain(secret, flattened, StringComparison.Ordinal);
        Assert.DoesNotContain(secret, string.Join('|', activityData), StringComparison.Ordinal);
        Assert.Contains(measurements, static item => item.Name == "rendezvous.http.requests");
        Assert.Contains(measurements, static item => item.Name == "rendezvous.udp.results");
        Assert.Contains(measurements, static item => item.Name == "rendezvous.limiter.drops");
        Assert.Contains(measurements, static item => item.Name == "rendezvous.queue.depth");
        Assert.Contains(measurements, static item => item.Name == "rendezvous.store.active_leases");
        Assert.Contains(measurements, static item => item.Name == "rendezvous.store.expiry_churn");
        Assert.Contains(measurements, static item => item.Name == "rendezvous.pairing.latency");
        Assert.All(measurements.SelectMany(static item => item.Tags), static tag =>
            Assert.Contains(tag.Key, AllowedTagKeys));
    }

    [Fact]
    public void MetricScrapeExpiresIdleStateAndReportsExpiryChurn()
    {
        EphemeralStateFixture fixture = new();
        Assert.True(fixture.Store.CreateListing(fixture.ListingCommand()).Succeeded);
        using RendezvousTelemetry telemetry = new(fixture.Store);
        List<Measurement> measurements = [];
        using MeterListener listener = new();
        listener.InstrumentPublished = (instrument, meterListener) =>
        {
            if (instrument.Meter.Name == RendezvousTelemetry.MeterName)
            {
                meterListener.EnableMeasurementEvents(instrument);
            }
        };
        listener.SetMeasurementEventCallback<long>((instrument, value, tags, _) =>
            measurements.Add(new(instrument.Name, value, Tags(tags))));
        listener.SetMeasurementEventCallback<int>((instrument, value, tags, _) =>
            measurements.Add(new(instrument.Name, value, Tags(tags))));
        listener.Start();

        listener.RecordObservableInstruments();
        Assert.Equal(
            1,
            Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value);

        fixture.Clock.Advance(TimeSpan.FromSeconds(61));
        measurements.Clear();
        listener.RecordObservableInstruments();
        Assert.Equal(
            0,
            Assert.Single(measurements, static item => item.Name == "rendezvous.store.active_leases").Value);
        Assert.True(Assert.Single(
            measurements,
            static item => item.Name == "rendezvous.store.expiry_churn").Value >= 1);
    }

    [Fact]
    public void AuditTrailFingerprintsIdentifiersEnforcesRetentionAndBoundsCapacity()
    {
        EphemeralStateFixture fixture = new();
        using RendezvousTelemetry telemetry = new(fixture.Store);
        CapturingLogger<AuditTrail> logger = new();
        ManualTimeProvider time = new(new DateTimeOffset(2026, 7, 16, 0, 0, 0, TimeSpan.Zero));
        AuditTrail audit = new(
            Options.Create(new AuditOptions { MaxEntries = 100, RetentionDays = 1 }),
            logger,
            telemetry,
            time);
        const string actor = "operator-secret-subject";
        const string target = "player-secret-subject";

        for (int index = 0; index < 101; index++)
        {
            audit.Record(actor, "revoke-principal", "succeeded", "principal", target, "safe-correlation");
        }

        IReadOnlyList<AuditEntry> bounded = audit.GetEntriesForTests();
        Assert.Equal(100, bounded.Count);
        Assert.All(bounded, entry =>
        {
            Assert.DoesNotContain(actor, entry.ToString(), StringComparison.Ordinal);
            Assert.DoesNotContain(target, entry.ToString(), StringComparison.Ordinal);
            Assert.NotEqual(actor, entry.ActorFingerprint);
            Assert.NotEqual(target, entry.TargetFingerprint);
        });
        Assert.DoesNotContain(actor, string.Join('|', logger.Messages), StringComparison.Ordinal);
        Assert.DoesNotContain(target, string.Join('|', logger.Messages), StringComparison.Ordinal);

        time.Advance(TimeSpan.FromDays(2));
        Assert.Empty(audit.GetEntriesForTests());
        Assert.Empty(audit.GetAggregateCounts());
        audit.Record(actor, "inspect-status", "succeeded", "service", "rendezvous", "safe-correlation");
        Assert.Single(audit.GetEntriesForTests());
        Assert.Equal(1, audit.GetAggregateCounts()["inspect-status:succeeded"]);
    }

    private static KeyValuePair<string, object?>[] Tags(
        ReadOnlySpan<KeyValuePair<string, object?>> tags) => tags.ToArray();

    private sealed record Measurement(
        string Name,
        double Value,
        KeyValuePair<string, object?>[] Tags);

    private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider
    {
        private DateTimeOffset _now = now;
        public override DateTimeOffset GetUtcNow() => _now;
        public void Advance(TimeSpan duration) => _now += duration;
    }

    private sealed class NoopIntroductionSink : INatIntroductionSink
    {
        public static NoopIntroductionSink Instance { get; } = new();
        public void Introduce(NatIntroductionPlan plan)
        {
        }
    }
}

internal sealed class CapturingLogger<T> : ILogger<T>
{
    public List<string> Messages { get; } = [];

    public IDisposable? BeginScope<TState>(TState state) where TState : notnull => null;
    public bool IsEnabled(LogLevel logLevel) => true;

    public void Log<TState>(
        LogLevel logLevel,
        EventId eventId,
        TState state,
        Exception? exception,
        Func<TState, Exception?, string> formatter) => Messages.Add(formatter(state, exception));
}

internal sealed class CapturingLoggerProvider : ILoggerProvider
{
    public ConcurrentQueue<string> Messages { get; } = new();

    public ILogger CreateLogger(string categoryName) => new Sink(Messages);

    public void Dispose() => GC.SuppressFinalize(this);

    private sealed class Sink(ConcurrentQueue<string> messages) : ILogger
    {
        public IDisposable BeginScope<TState>(TState state) where TState : notnull => Scope.Instance;
        public bool IsEnabled(LogLevel logLevel) => true;

        public void Log<TState>(
            LogLevel logLevel,
            EventId eventId,
            TState state,
            Exception? exception,
            Func<TState, Exception?, string> formatter) => messages.Enqueue(formatter(state, exception));
    }

    private sealed class Scope : IDisposable
    {
        public static Scope Instance { get; } = new();
        public void Dispose()
        {
        }
    }
}