feat(server): add observability and operator controls (#16)

2026-07-16 13:22:16 +02:00
parent 88ef946af5
commit be732de7c9
31 changed files with 4149 additions and 140 deletions
@@ -0,0 +1,126 @@
+using System.Diagnostics;
+using System.Diagnostics.Metrics;
+using FinalFactory.Rendezvous.Server.State;
+
+namespace FinalFactory.Rendezvous.Server.Observability;
+
+internal sealed class RendezvousTelemetry : IDisposable
+{
+    public const string MeterName = "FinalFactory.Rendezvous";
+    public const string ActivitySourceName = "FinalFactory.Rendezvous.Server";
+
+    private readonly InMemoryEphemeralRendezvousStore _store;
+    private readonly Meter _meter = new(MeterName, "1.0.0");
+    private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0");
+    private readonly Counter<long> _httpRequests;
+    private readonly Histogram<double> _httpDuration;
+    private readonly Counter<long> _udpResults;
+    private readonly Histogram<double> _udpDuration;
+    private readonly Counter<long> _limiterDrops;
+    private readonly Counter<long> _auditEvents;
+    private readonly Counter<long> _connectionOutcomes;
+    private readonly Counter<long> _operatorAuthentication;
+    private readonly Histogram<double> _pairingLatency;
+
+    public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store)
+    {
+        _store = store;
+        _httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests");
+        _httpDuration = _meter.CreateHistogram<double>(
+            "rendezvous.http.duration",
+            "ms");
+        _udpResults = _meter.CreateCounter<long>("rendezvous.udp.results");
+        _udpDuration = _meter.CreateHistogram<double>(
+            "rendezvous.udp.duration",
+            "ms");
+        _limiterDrops = _meter.CreateCounter<long>("rendezvous.limiter.drops");
+        _auditEvents = _meter.CreateCounter<long>("rendezvous.audit.events");
+        _connectionOutcomes = _meter.CreateCounter<long>("rendezvous.connection.outcomes");
+        _operatorAuthentication = _meter.CreateCounter<long>("rendezvous.operator.authentication");
+        _pairingLatency = _meter.CreateHistogram<double>(
+            "rendezvous.pairing.latency",
+            "ms");
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_listings",
+            () => _store.GetMetricsSnapshot().ActiveListings);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_leases",
+            () => _store.GetMetricsSnapshot().ActiveListings);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.active_attempts",
+            () => _store.GetMetricsSnapshot().ActiveJoinAttempts);
+        _meter.CreateObservableGauge(
+            "rendezvous.queue.depth",
+            () => _store.GetMetricsSnapshot().ActiveJoinAttempts);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.replay_markers",
+            () => _store.GetMetricsSnapshot().ReplayMarkers);
+        _meter.CreateObservableGauge(
+            "rendezvous.store.available",
+            () => _store.GetMetricsSnapshot().IsAvailable ? 1 : 0);
+        _meter.CreateObservableCounter(
+            "rendezvous.store.expiry_churn",
+            () => _store.GetMetricsSnapshot().ExpiryChurn);
+    }
+
+    public Activity? StartActivity(string name, ActivityKind kind = ActivityKind.Internal) =>
+        _activities.StartActivity(name, kind);
+
+    public void RecordHttp(string operation, int statusCode, double elapsedMilliseconds)
+    {
+        TagList tags = new()
+        {
+            { "operation", operation },
+            { "status_code", statusCode },
+        };
+        _httpRequests.Add(1, tags);
+        _httpDuration.Record(elapsedMilliseconds, tags);
+    }
+
+    public void RecordUdp(string operation, string result, double elapsedMilliseconds)
+    {
+        TagList tags = new()
+        {
+            { "operation", operation },
+            { "result", result },
+        };
+        _udpResults.Add(1, tags);
+        _udpDuration.Record(elapsedMilliseconds, tags);
+    }
+
+    public void RecordLimiterDrop(string transport, string partition) =>
+        _limiterDrops.Add(1, new TagList
+        {
+            { "transport", transport },
+            { "partition", partition },
+        });
+
+    public void RecordAudit(string action, string result) =>
+        _auditEvents.Add(1, new TagList
+        {
+            { "action", action },
+            { "result", result },
+        });
+
+    public void RecordConnectionOutcome(string outcome, string elapsedBucket) =>
+        _connectionOutcomes.Add(1, new TagList
+        {
+            { "outcome", outcome },
+            { "elapsed_bucket", elapsedBucket },
+        });
+
+    public void RecordOperatorAuthentication(string result) =>
+        _operatorAuthentication.Add(1, new TagList
+        {
+            { "result", result },
+        });
+
+    public void RecordPairingLatency(double elapsedMilliseconds) =>
+        _pairingLatency.Record(elapsedMilliseconds);
+
+    public void Dispose()
+    {
+        _activities.Dispose();
+        _meter.Dispose();
+    }
+}