feat(observability): add diagnostic dashboards (#27)
This commit is contained in:
KyuubiYoru
@@ -143,6 +143,39 @@ internal sealed class SessionChangeJournal
|
||||
|
||||
public SessionChangeJournalOptions Options => _options;
|
||||
|
||||
public int ReplayCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _changes.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int SubscriberCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _subscribers;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public int SubscribedTenantCount
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _subscribersByTenant.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public long CurrentRevision
|
||||
{
|
||||
get
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
const CONTRACT_VERSION = 1;
|
||||
const BROWSER_PAGE_LIMIT = 100;
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
const STREAM_FAILURE_LIMIT = 3;
|
||||
|
||||
export class SessionProjection {
|
||||
constructor(maximumSessions = 100) {
|
||||
if (!Number.isInteger(maximumSessions) || maximumSessions < 1 || maximumSessions > 500) {
|
||||
throw new RangeError("maximumSessions must be between 1 and 500");
|
||||
}
|
||||
this.maximumSessions = maximumSessions;
|
||||
this.entries = new Map();
|
||||
this.cursor = "";
|
||||
this.hasMore = false;
|
||||
}
|
||||
|
||||
replace(items, cursor, hasMore = false, updatedAt = Date.now()) {
|
||||
if (!Array.isArray(items) || items.length > this.maximumSessions) {
|
||||
return "overflow";
|
||||
}
|
||||
const next = new Map();
|
||||
for (const session of items) {
|
||||
if (!isPublicSession(session) || next.has(session.listingId)) {
|
||||
return "invalid";
|
||||
}
|
||||
next.set(session.listingId, { session, updatedAt });
|
||||
}
|
||||
this.entries = next;
|
||||
this.cursor = validCursor(cursor) ? cursor : "";
|
||||
this.hasMore = Boolean(hasMore);
|
||||
return "applied";
|
||||
}
|
||||
|
||||
apply(event, updatedAt = Date.now()) {
|
||||
if (!event || event.contractVersion !== CONTRACT_VERSION || !validCursor(event.cursor)) {
|
||||
return "invalid";
|
||||
}
|
||||
if (event.kind === "sessionUpsert" && isPublicSession(event.session) && event.listingId == null) {
|
||||
if (!this.entries.has(event.session.listingId)
|
||||
&& this.entries.size >= this.maximumSessions) {
|
||||
return "overflow";
|
||||
}
|
||||
this.entries.set(event.session.listingId, { session: event.session, updatedAt });
|
||||
this.cursor = event.cursor;
|
||||
return "applied";
|
||||
}
|
||||
if (event.kind === "sessionRemove"
|
||||
&& typeof event.listingId === "string"
|
||||
&& event.listingId.length > 0
|
||||
&& event.session == null) {
|
||||
this.entries.delete(event.listingId);
|
||||
this.cursor = event.cursor;
|
||||
return this.hasMore ? "refresh" : "applied";
|
||||
}
|
||||
if (event.kind === "keepalive" && event.session == null && event.listingId == null) {
|
||||
this.cursor = event.cursor;
|
||||
return "keepalive";
|
||||
}
|
||||
if (event.kind === "reset" && event.session == null && event.listingId == null) {
|
||||
this.cursor = event.cursor;
|
||||
return "reset";
|
||||
}
|
||||
return "invalid";
|
||||
}
|
||||
|
||||
sessions() {
|
||||
return [...this.entries.values()]
|
||||
.sort((left, right) => left.session.listingId.localeCompare(right.session.listingId));
|
||||
}
|
||||
}
|
||||
|
||||
export function safeText(value, maximumLength = 160) {
|
||||
const text = typeof value === "string" ? value : String(value ?? "");
|
||||
const visible = text.replace(/[\u0000-\u001f\u007f]/gu, "�");
|
||||
return visible.length <= maximumLength
|
||||
? visible
|
||||
: `${visible.slice(0, Math.max(0, maximumLength - 1))}…`;
|
||||
}
|
||||
|
||||
export function buildBrowseUrl(filter) {
|
||||
validateFilter(filter);
|
||||
const query = new URLSearchParams({
|
||||
contractVersion: String(CONTRACT_VERSION),
|
||||
gameId: filter.gameId,
|
||||
environmentId: filter.environmentId,
|
||||
protocolVersion: String(filter.protocolVersion),
|
||||
regionId: filter.regionId,
|
||||
pageSize: String(Math.min(BROWSER_PAGE_LIMIT, filter.pageSize ?? BROWSER_PAGE_LIMIT)),
|
||||
excludeFull: String(Boolean(filter.excludeFull)),
|
||||
});
|
||||
return `/v1/sessions?${query}`;
|
||||
}
|
||||
|
||||
export function buildStreamUrl(filter, cursor) {
|
||||
validateFilter(filter);
|
||||
if (!validCursor(cursor)) {
|
||||
throw new TypeError("A bounded stream cursor is required");
|
||||
}
|
||||
const query = new URLSearchParams({
|
||||
contractVersion: String(CONTRACT_VERSION),
|
||||
gameId: filter.gameId,
|
||||
environmentId: filter.environmentId,
|
||||
protocolVersion: String(filter.protocolVersion),
|
||||
regionId: filter.regionId,
|
||||
excludeFull: String(Boolean(filter.excludeFull)),
|
||||
streamCursor: cursor,
|
||||
});
|
||||
return `/v1/sessions/stream?${query}`;
|
||||
}
|
||||
|
||||
export function chooseSnapshotTransport(hasMore, pollingOnly) {
|
||||
if (pollingOnly) {
|
||||
return "pollingOnly";
|
||||
}
|
||||
return hasMore ? "boundedPolling" : "stream";
|
||||
}
|
||||
|
||||
function validCursor(value) {
|
||||
return typeof value === "string"
|
||||
&& value.length > 0
|
||||
&& value.length <= 1024
|
||||
&& /^[\x21-\x7e]+$/u.test(value);
|
||||
}
|
||||
|
||||
function isPublicSession(session) {
|
||||
return session != null
|
||||
&& session.contractVersion === CONTRACT_VERSION
|
||||
&& typeof session.listingId === "string"
|
||||
&& session.listingId.length > 0
|
||||
&& typeof session.gameId === "string"
|
||||
&& typeof session.environmentId === "string"
|
||||
&& typeof session.regionId === "string"
|
||||
&& Number.isInteger(session.protocolVersion)
|
||||
&& session.protocolVersion > 0
|
||||
&& typeof session.buildVersion === "string"
|
||||
&& typeof session.displayName === "string"
|
||||
&& session.visibility === "public"
|
||||
&& session.capacity != null
|
||||
&& Number.isInteger(session.capacity.currentPlayers)
|
||||
&& Number.isInteger(session.capacity.maximumPlayers)
|
||||
&& session.capacity.currentPlayers >= 0
|
||||
&& session.capacity.maximumPlayers >= 1
|
||||
&& session.capacity.currentPlayers <= session.capacity.maximumPlayers
|
||||
&& session.metadata != null
|
||||
&& typeof session.metadata === "object"
|
||||
&& !Array.isArray(session.metadata);
|
||||
}
|
||||
|
||||
function validateFilter(filter) {
|
||||
if (!filter
|
||||
|| typeof filter.gameId !== "string"
|
||||
|| typeof filter.environmentId !== "string"
|
||||
|| typeof filter.regionId !== "string"
|
||||
|| !Number.isInteger(filter.protocolVersion)
|
||||
|| filter.protocolVersion <= 0) {
|
||||
throw new TypeError("The selected diagnostic filter is invalid");
|
||||
}
|
||||
}
|
||||
|
||||
function startDashboard() {
|
||||
const elements = {
|
||||
form: document.querySelector("#filters"),
|
||||
game: document.querySelector("#game-filter"),
|
||||
environment: document.querySelector("#environment-filter"),
|
||||
protocol: document.querySelector("#protocol-filter"),
|
||||
region: document.querySelector("#region-filter"),
|
||||
capacity: document.querySelector("#capacity-filter"),
|
||||
error: document.querySelector("#filter-error"),
|
||||
reconnect: document.querySelector("#reconnect-button"),
|
||||
reset: document.querySelector("#reset-button"),
|
||||
poll: document.querySelector("#poll-button"),
|
||||
streamStatus: document.querySelector("#stream-status"),
|
||||
transportState: document.querySelector("#transport-state"),
|
||||
projectionState: document.querySelector("#projection-state"),
|
||||
lastUpdate: document.querySelector("#last-update"),
|
||||
sessionCount: document.querySelector("#session-count"),
|
||||
results: document.querySelector(".results-panel"),
|
||||
resultsSummary: document.querySelector("#results-summary"),
|
||||
list: document.querySelector("#session-list"),
|
||||
empty: document.querySelector("#empty-state"),
|
||||
};
|
||||
|
||||
const state = {
|
||||
configuration: null,
|
||||
projection: null,
|
||||
activeFilter: null,
|
||||
source: null,
|
||||
pollingTimer: null,
|
||||
requestController: null,
|
||||
streamFailures: 0,
|
||||
pollingOnly: false,
|
||||
renderPending: false,
|
||||
lastSuccess: 0,
|
||||
};
|
||||
|
||||
function setStatus(kind, message, transport = message) {
|
||||
document.body.dataset.streamState = kind;
|
||||
elements.streamStatus.textContent = message;
|
||||
elements.transportState.textContent = transport;
|
||||
}
|
||||
|
||||
function setFormError(message = "") {
|
||||
elements.error.textContent = message;
|
||||
elements.error.hidden = message.length === 0;
|
||||
}
|
||||
|
||||
function populate(select, values, previous) {
|
||||
select.replaceChildren();
|
||||
for (const value of values) {
|
||||
const option = document.createElement("option");
|
||||
option.value = String(value);
|
||||
option.textContent = safeText(value, 80);
|
||||
select.append(option);
|
||||
}
|
||||
if (values.some((value) => String(value) === previous)) {
|
||||
select.value = previous;
|
||||
}
|
||||
}
|
||||
|
||||
function selectedScope() {
|
||||
return state.configuration?.scopes.find((scope) =>
|
||||
scope.gameId === elements.game.value
|
||||
&& scope.environmentId === elements.environment.value) ?? null;
|
||||
}
|
||||
|
||||
function updateEnvironmentOptions() {
|
||||
const prior = elements.environment.value;
|
||||
const environments = state.configuration.scopes
|
||||
.filter((scope) => scope.gameId === elements.game.value)
|
||||
.map((scope) => scope.environmentId);
|
||||
populate(elements.environment, environments, prior);
|
||||
updateScopeOptions();
|
||||
}
|
||||
|
||||
function updateScopeOptions() {
|
||||
const scope = selectedScope();
|
||||
populate(elements.protocol, scope?.protocolVersions ?? [], elements.protocol.value);
|
||||
populate(elements.region, scope?.regions ?? [], elements.region.value);
|
||||
}
|
||||
|
||||
function currentFilter() {
|
||||
const scope = selectedScope();
|
||||
const protocolVersion = Number(elements.protocol.value);
|
||||
if (!scope
|
||||
|| !scope.protocolVersions.includes(protocolVersion)
|
||||
|| !scope.regions.includes(elements.region.value)) {
|
||||
throw new TypeError("Choose one of the configured game, environment, protocol, and region combinations.");
|
||||
}
|
||||
return {
|
||||
gameId: scope.gameId,
|
||||
environmentId: scope.environmentId,
|
||||
protocolVersion,
|
||||
regionId: elements.region.value,
|
||||
excludeFull: elements.capacity.value === "open",
|
||||
pageSize: Math.min(BROWSER_PAGE_LIMIT, state.configuration.maximumRenderedSessions),
|
||||
};
|
||||
}
|
||||
|
||||
function stopLiveWork() {
|
||||
if (state.source) {
|
||||
state.source.close();
|
||||
state.source = null;
|
||||
}
|
||||
if (state.pollingTimer) {
|
||||
clearTimeout(state.pollingTimer);
|
||||
state.pollingTimer = null;
|
||||
}
|
||||
if (state.requestController) {
|
||||
state.requestController.abort();
|
||||
state.requestController = null;
|
||||
}
|
||||
}
|
||||
|
||||
function markSuccess(message) {
|
||||
state.lastSuccess = Date.now();
|
||||
elements.lastUpdate.dateTime = new Date(state.lastSuccess).toISOString();
|
||||
elements.lastUpdate.textContent = "Just now";
|
||||
elements.resultsSummary.textContent = message;
|
||||
}
|
||||
|
||||
function scheduleRender() {
|
||||
if (state.renderPending) {
|
||||
return;
|
||||
}
|
||||
state.renderPending = true;
|
||||
requestAnimationFrame(() => {
|
||||
state.renderPending = false;
|
||||
renderProjection();
|
||||
});
|
||||
}
|
||||
|
||||
function appendDetail(list, term, description) {
|
||||
const dt = document.createElement("dt");
|
||||
dt.textContent = term;
|
||||
const dd = document.createElement("dd");
|
||||
dd.textContent = safeText(description, 160);
|
||||
list.append(dt, dd);
|
||||
}
|
||||
|
||||
function sessionCard(entry) {
|
||||
const session = entry.session;
|
||||
const article = document.createElement("article");
|
||||
article.className = "session-card";
|
||||
article.setAttribute("role", "listitem");
|
||||
article.dataset.updatedAt = String(entry.updatedAt);
|
||||
|
||||
const heading = document.createElement("div");
|
||||
heading.className = "card-heading";
|
||||
const title = document.createElement("h3");
|
||||
title.textContent = safeText(session.displayName, 120);
|
||||
const region = document.createElement("span");
|
||||
region.className = "region-badge";
|
||||
region.textContent = safeText(session.regionId, 48);
|
||||
heading.append(title, region);
|
||||
|
||||
const details = document.createElement("dl");
|
||||
details.className = "session-detail";
|
||||
appendDetail(details, "Build", session.buildVersion);
|
||||
appendDetail(details, "Protocol", session.protocolVersion);
|
||||
appendDetail(details, "Visibility", "Public");
|
||||
appendDetail(details, "Presence", "Recently verified");
|
||||
|
||||
const capacity = document.createElement("div");
|
||||
capacity.className = "capacity-row";
|
||||
const capacityCopy = document.createElement("div");
|
||||
capacityCopy.className = "capacity-copy";
|
||||
const capacityLabel = document.createElement("span");
|
||||
capacityLabel.textContent = "Advisory capacity";
|
||||
const capacityValue = document.createElement("span");
|
||||
capacityValue.textContent = `${session.capacity.currentPlayers} / ${session.capacity.maximumPlayers}`;
|
||||
capacityCopy.append(capacityLabel, capacityValue);
|
||||
const meter = document.createElement("meter");
|
||||
meter.min = 0;
|
||||
meter.max = session.capacity.maximumPlayers;
|
||||
meter.value = session.capacity.currentPlayers;
|
||||
meter.setAttribute("aria-label", `Advisory capacity ${capacityValue.textContent}`);
|
||||
capacity.append(capacityCopy, meter);
|
||||
|
||||
const metadata = document.createElement("ul");
|
||||
metadata.className = "metadata-list";
|
||||
for (const [key, value] of Object.entries(session.metadata).slice(0, 16)) {
|
||||
const item = document.createElement("li");
|
||||
item.textContent = `${safeText(key, 48)}: ${safeText(value, 96)}`;
|
||||
metadata.append(item);
|
||||
}
|
||||
if (metadata.childElementCount === 0) {
|
||||
const item = document.createElement("li");
|
||||
item.textContent = "No public metadata";
|
||||
metadata.append(item);
|
||||
}
|
||||
|
||||
const updated = document.createElement("p");
|
||||
updated.className = "updated-age";
|
||||
updated.textContent = "Updated just now";
|
||||
article.append(heading, details, capacity, metadata, updated);
|
||||
return article;
|
||||
}
|
||||
|
||||
function renderProjection() {
|
||||
const entries = state.projection?.sessions() ?? [];
|
||||
const fragment = document.createDocumentFragment();
|
||||
for (const entry of entries) {
|
||||
fragment.append(sessionCard(entry));
|
||||
}
|
||||
elements.list.replaceChildren(fragment);
|
||||
elements.sessionCount.textContent = String(entries.length);
|
||||
elements.empty.hidden = entries.length !== 0;
|
||||
elements.projectionState.textContent = state.projection?.hasMore
|
||||
? "Bounded snapshot; polling"
|
||||
: "Snapshot plus ordered deltas";
|
||||
elements.results.setAttribute("aria-busy", "false");
|
||||
}
|
||||
|
||||
function updateAges() {
|
||||
const now = Date.now();
|
||||
if (state.lastSuccess > 0) {
|
||||
const age = Math.max(0, Math.floor((now - state.lastSuccess) / 1000));
|
||||
elements.lastUpdate.textContent = age < 5 ? "Just now" : `${age} seconds ago`;
|
||||
}
|
||||
for (const card of elements.list.querySelectorAll(".session-card")) {
|
||||
const age = Math.max(0, Math.floor((now - Number(card.dataset.updatedAt)) / 1000));
|
||||
const copy = card.querySelector(".updated-age");
|
||||
copy.textContent = age < 5 ? "Updated just now" : `Updated ${age} seconds ago`;
|
||||
}
|
||||
}
|
||||
|
||||
async function readJson(url) {
|
||||
const controller = new AbortController();
|
||||
state.requestController = controller;
|
||||
const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
cache: "no-store",
|
||||
credentials: "same-origin",
|
||||
headers: { Accept: "application/json" },
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`The service returned HTTP ${response.status}.`);
|
||||
}
|
||||
return await response.json();
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
if (state.requestController === controller) {
|
||||
state.requestController = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function schedulePolling() {
|
||||
if (state.pollingTimer) {
|
||||
clearTimeout(state.pollingTimer);
|
||||
}
|
||||
state.pollingTimer = setTimeout(
|
||||
() => fetchSnapshot("poll").catch(showServiceError),
|
||||
state.configuration.pollIntervalSeconds * 1000,
|
||||
);
|
||||
}
|
||||
|
||||
function usePolling(message = "Polling fallback active") {
|
||||
if (state.source) {
|
||||
state.source.close();
|
||||
state.source = null;
|
||||
}
|
||||
setStatus("polling", message, "Polling fallback");
|
||||
schedulePolling();
|
||||
}
|
||||
|
||||
function showServiceError(error) {
|
||||
const message = error?.name === "AbortError"
|
||||
? "The service did not answer within the request deadline."
|
||||
: safeText(error?.message || "The service is unavailable.", 200);
|
||||
setStatus("error", "Service unavailable", "Unavailable");
|
||||
elements.resultsSummary.textContent = `${message} Retrying with bounded polling.`;
|
||||
elements.results.setAttribute("aria-busy", "false");
|
||||
usePolling("Service unavailable; polling retry scheduled");
|
||||
}
|
||||
|
||||
async function fetchSnapshot(reason = "manual") {
|
||||
if (!state.activeFilter) {
|
||||
return;
|
||||
}
|
||||
if (state.requestController) {
|
||||
state.requestController.abort();
|
||||
}
|
||||
elements.results.setAttribute("aria-busy", "true");
|
||||
if (reason !== "poll") {
|
||||
setStatus("reconnecting", "Loading a fresh snapshot", "Snapshot request");
|
||||
}
|
||||
const response = await readJson(buildBrowseUrl(state.activeFilter));
|
||||
if (response.contractVersion !== CONTRACT_VERSION
|
||||
|| !Array.isArray(response.items)
|
||||
|| !validCursor(response.streamCursor)) {
|
||||
throw new Error("The service returned an invalid browser snapshot.");
|
||||
}
|
||||
const outcome = state.projection.replace(
|
||||
response.items,
|
||||
response.streamCursor,
|
||||
Boolean(response.nextCursor),
|
||||
);
|
||||
if (outcome !== "applied") {
|
||||
throw new Error("The bounded browser snapshot could not be applied safely.");
|
||||
}
|
||||
scheduleRender();
|
||||
markSuccess(`${response.items.length} public session${response.items.length === 1 ? "" : "s"} in the fresh snapshot.`);
|
||||
const transport = chooseSnapshotTransport(Boolean(response.nextCursor), state.pollingOnly);
|
||||
if (transport !== "stream") {
|
||||
if (transport === "pollingOnly") {
|
||||
usePolling("Polling only selected");
|
||||
return;
|
||||
}
|
||||
usePolling("More sessions exist than this bounded view; polling keeps it authoritative");
|
||||
return;
|
||||
}
|
||||
connectStream(response.streamCursor);
|
||||
}
|
||||
|
||||
function handleStreamEvent(serialized, expectedKind) {
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(serialized);
|
||||
} catch {
|
||||
usePolling("Invalid stream data; polling fallback active");
|
||||
return;
|
||||
}
|
||||
if (event.kind !== expectedKind) {
|
||||
usePolling("Unexpected stream event; polling fallback active");
|
||||
return;
|
||||
}
|
||||
const outcome = state.projection.apply(event);
|
||||
if (outcome === "invalid" || outcome === "overflow" || outcome === "refresh") {
|
||||
fetchSnapshot("stream-recovery").catch(showServiceError);
|
||||
return;
|
||||
}
|
||||
if (outcome === "reset") {
|
||||
setStatus("reset", "Cursor reset; refreshing snapshot", "Cursor reset");
|
||||
fetchSnapshot("reset").catch(showServiceError);
|
||||
return;
|
||||
}
|
||||
if (outcome === "keepalive") {
|
||||
markSuccess("Live connection healthy; no listing changes.");
|
||||
return;
|
||||
}
|
||||
state.streamFailures = 0;
|
||||
scheduleRender();
|
||||
markSuccess(expectedKind === "sessionUpsert"
|
||||
? "Applied a live session add or update."
|
||||
: "Applied a live session removal.");
|
||||
}
|
||||
|
||||
function connectStream(cursor, corrupt = false) {
|
||||
if (state.pollingTimer) {
|
||||
clearTimeout(state.pollingTimer);
|
||||
state.pollingTimer = null;
|
||||
}
|
||||
if (state.source) {
|
||||
state.source.close();
|
||||
}
|
||||
const selectedCursor = corrupt
|
||||
? `${cursor.slice(0, -1)}${cursor.endsWith("a") ? "b" : "a"}`
|
||||
: cursor;
|
||||
setStatus(corrupt ? "reset" : "reconnecting",
|
||||
corrupt ? "Testing cursor reset" : "Connecting live updates",
|
||||
corrupt ? "Cursor reset test" : "SSE reconnecting");
|
||||
const source = new EventSource(buildStreamUrl(state.activeFilter, selectedCursor));
|
||||
state.source = source;
|
||||
source.addEventListener("open", () => {
|
||||
if (state.source !== source) {
|
||||
return;
|
||||
}
|
||||
state.streamFailures = 0;
|
||||
setStatus("connected", "Live updates connected", "SSE live");
|
||||
});
|
||||
for (const [name, kind] of [
|
||||
["session_upsert", "sessionUpsert"],
|
||||
["session_remove", "sessionRemove"],
|
||||
["reset", "reset"],
|
||||
["keepalive", "keepalive"],
|
||||
]) {
|
||||
source.addEventListener(name, (message) => {
|
||||
if (state.source === source) {
|
||||
handleStreamEvent(message.data, kind);
|
||||
}
|
||||
});
|
||||
}
|
||||
source.addEventListener("error", () => {
|
||||
if (state.source !== source) {
|
||||
return;
|
||||
}
|
||||
state.streamFailures += 1;
|
||||
if (state.streamFailures >= STREAM_FAILURE_LIMIT || source.readyState === EventSource.CLOSED) {
|
||||
usePolling("Live stream unavailable; polling fallback active");
|
||||
} else {
|
||||
setStatus("reconnecting", "Live stream interrupted; reconnecting", "SSE reconnecting");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function loadConfiguration() {
|
||||
const configuration = await readJson("/diagnostics/config.json");
|
||||
if (configuration.contractVersion !== CONTRACT_VERSION
|
||||
|| !Array.isArray(configuration.scopes)
|
||||
|| configuration.scopes.length < 1
|
||||
|| !Number.isInteger(configuration.pollIntervalSeconds)
|
||||
|| !Number.isInteger(configuration.maximumRenderedSessions)) {
|
||||
throw new Error("The diagnostic configuration is invalid.");
|
||||
}
|
||||
state.configuration = configuration;
|
||||
state.projection = new SessionProjection(configuration.maximumRenderedSessions);
|
||||
populate(elements.game, [...new Set(configuration.scopes.map((scope) => scope.gameId))], "");
|
||||
updateEnvironmentOptions();
|
||||
state.activeFilter = currentFilter();
|
||||
await fetchSnapshot("startup");
|
||||
}
|
||||
|
||||
elements.game.addEventListener("change", updateEnvironmentOptions);
|
||||
elements.environment.addEventListener("change", updateScopeOptions);
|
||||
elements.form.addEventListener("submit", (event) => {
|
||||
event.preventDefault();
|
||||
try {
|
||||
setFormError();
|
||||
stopLiveWork();
|
||||
state.pollingOnly = false;
|
||||
state.activeFilter = currentFilter();
|
||||
state.projection = new SessionProjection(state.configuration.maximumRenderedSessions);
|
||||
fetchSnapshot("filter").catch(showServiceError);
|
||||
} catch (error) {
|
||||
setFormError(safeText(error.message, 200));
|
||||
}
|
||||
});
|
||||
elements.reconnect.addEventListener("click", () => {
|
||||
if (state.projection?.cursor) {
|
||||
state.pollingOnly = false;
|
||||
connectStream(state.projection.cursor);
|
||||
}
|
||||
});
|
||||
elements.reset.addEventListener("click", () => {
|
||||
if (state.projection?.cursor) {
|
||||
state.pollingOnly = false;
|
||||
connectStream(state.projection.cursor, true);
|
||||
}
|
||||
});
|
||||
elements.poll.addEventListener("click", () => {
|
||||
state.pollingOnly = true;
|
||||
usePolling("Polling only selected deliberately");
|
||||
fetchSnapshot("poll").catch(showServiceError);
|
||||
});
|
||||
window.addEventListener("pagehide", stopLiveWork, { once: true });
|
||||
setInterval(updateAges, 5000);
|
||||
loadConfiguration().catch(showServiceError);
|
||||
}
|
||||
|
||||
if (typeof document !== "undefined") {
|
||||
startDashboard();
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="dark">
|
||||
<title>Session diagnostics — Rendezvous</title>
|
||||
<link rel="stylesheet" href="/diagnostics/styles.css">
|
||||
<script type="module" src="/diagnostics/app.mjs"></script>
|
||||
</head>
|
||||
<body>
|
||||
<a class="skip-link" href="#main-content">Skip to session results</a>
|
||||
<header class="site-header">
|
||||
<div>
|
||||
<p class="eyebrow">Final Factory infrastructure</p>
|
||||
<h1>Rendezvous session diagnostics</h1>
|
||||
<p class="lede">A read-only view of public session discovery. Capacity is advisory; joining always revalidates current state.</p>
|
||||
</div>
|
||||
<div class="connection-state" aria-live="polite" aria-atomic="true">
|
||||
<span class="status-dot" aria-hidden="true"></span>
|
||||
<span id="stream-status">Loading configuration</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main id="main-content" tabindex="-1">
|
||||
<section class="filter-panel" aria-labelledby="filter-heading">
|
||||
<div class="section-heading">
|
||||
<div>
|
||||
<p class="eyebrow">Public browser scope</p>
|
||||
<h2 id="filter-heading">Choose a configured session view</h2>
|
||||
</div>
|
||||
<p class="boundary-note">This page has no join, publish, or operator authority.</p>
|
||||
</div>
|
||||
|
||||
<form id="filters" novalidate>
|
||||
<div class="filter-grid">
|
||||
<label>
|
||||
<span>Game</span>
|
||||
<select id="game-filter" name="game" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Environment</span>
|
||||
<select id="environment-filter" name="environment" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Protocol</span>
|
||||
<select id="protocol-filter" name="protocol" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Region</span>
|
||||
<select id="region-filter" name="region" required></select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Availability</span>
|
||||
<select id="capacity-filter" name="capacity">
|
||||
<option value="open">Open slots only</option>
|
||||
<option value="all">Include full sessions</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
<span>Visibility</span>
|
||||
<select id="visibility-filter" name="visibility" disabled>
|
||||
<option value="public">Public only</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<p id="filter-error" class="form-error" role="alert" hidden></p>
|
||||
<div class="button-row">
|
||||
<button class="button primary" type="submit">Apply filters</button>
|
||||
<button class="button" id="reconnect-button" type="button">Reconnect now</button>
|
||||
<button class="button" id="reset-button" type="button">Test reset recovery</button>
|
||||
<button class="button" id="poll-button" type="button">Use polling only</button>
|
||||
</div>
|
||||
</form>
|
||||
</section>
|
||||
|
||||
<section class="summary-panel" aria-labelledby="summary-heading">
|
||||
<h2 id="summary-heading" class="visually-hidden">Current diagnostic state</h2>
|
||||
<dl class="summary-grid">
|
||||
<div>
|
||||
<dt>Sessions shown</dt>
|
||||
<dd id="session-count">0</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Transport</dt>
|
||||
<dd id="transport-state">Starting</dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Last successful update</dt>
|
||||
<dd><time id="last-update">Not yet</time></dd>
|
||||
</div>
|
||||
<div>
|
||||
<dt>Projection</dt>
|
||||
<dd id="projection-state">Waiting for snapshot</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<section class="results-panel" aria-labelledby="results-heading" aria-busy="true">
|
||||
<div class="section-heading results-heading">
|
||||
<div>
|
||||
<p class="eyebrow">Bounded public projection</p>
|
||||
<h2 id="results-heading">Available sessions</h2>
|
||||
</div>
|
||||
<p id="results-summary" role="status" aria-live="polite">Loading a fresh snapshot…</p>
|
||||
</div>
|
||||
<div id="session-list" class="session-grid" role="list"></div>
|
||||
<div id="empty-state" class="empty-state" hidden>
|
||||
<h3>No compatible public sessions</h3>
|
||||
<p>The service answered successfully, but this configured filter currently has no listings.</p>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<footer>
|
||||
<p>Read-only diagnostics · no credentials stored · no gameplay or administration</p>
|
||||
</footer>
|
||||
<noscript>This diagnostic requires JavaScript to consume the public snapshot and event stream.</noscript>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,505 @@
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: #071019;
|
||||
--surface: #101c28;
|
||||
--surface-raised: #172737;
|
||||
--border: #375066;
|
||||
--text: #f2f7fb;
|
||||
--muted: #b6c8d6;
|
||||
--accent: #55d7b5;
|
||||
--accent-strong: #7ce9cc;
|
||||
--accent-ink: #04231c;
|
||||
--warning: #ffd479;
|
||||
--danger: #ff9f9f;
|
||||
--focus: #ffd479;
|
||||
--radius: 0.75rem;
|
||||
--space-1: 0.5rem;
|
||||
--space-2: 0.75rem;
|
||||
--space-3: 1rem;
|
||||
--space-4: 1.5rem;
|
||||
--space-5: 2rem;
|
||||
--space-6: 3rem;
|
||||
font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
font-size: 100%;
|
||||
line-height: 1.55;
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
html {
|
||||
min-width: 20rem;
|
||||
background: var(--bg);
|
||||
}
|
||||
|
||||
body {
|
||||
min-height: 100vh;
|
||||
margin: 0;
|
||||
color: var(--text);
|
||||
background:
|
||||
radial-gradient(circle at 10% -10%, rgb(36 103 104 / 35%), transparent 32rem),
|
||||
linear-gradient(180deg, #09141f 0%, var(--bg) 50%);
|
||||
}
|
||||
|
||||
button,
|
||||
select {
|
||||
font: inherit;
|
||||
}
|
||||
|
||||
button,
|
||||
select,
|
||||
a {
|
||||
-webkit-tap-highlight-color: transparent;
|
||||
}
|
||||
|
||||
:focus-visible {
|
||||
outline: 0.2rem solid var(--focus);
|
||||
outline-offset: 0.2rem;
|
||||
}
|
||||
|
||||
.skip-link {
|
||||
position: fixed;
|
||||
z-index: 10;
|
||||
top: var(--space-2);
|
||||
left: var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
color: #071019;
|
||||
background: var(--focus);
|
||||
border-radius: 0.4rem;
|
||||
font-weight: 800;
|
||||
transform: translateY(-200%);
|
||||
}
|
||||
|
||||
.skip-link:focus {
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
.site-header,
|
||||
main,
|
||||
footer {
|
||||
width: min(80rem, calc(100% - 2rem));
|
||||
margin-inline: auto;
|
||||
}
|
||||
|
||||
.site-header {
|
||||
display: flex;
|
||||
align-items: end;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-5);
|
||||
padding-block: var(--space-6) var(--space-5);
|
||||
}
|
||||
|
||||
h1,
|
||||
h2,
|
||||
h3,
|
||||
p {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
h1 {
|
||||
max-width: 18ch;
|
||||
margin-bottom: var(--space-2);
|
||||
font-size: clamp(2rem, 6vw, 4.25rem);
|
||||
line-height: 1.02;
|
||||
letter-spacing: -0.045em;
|
||||
}
|
||||
|
||||
h2 {
|
||||
margin-bottom: var(--space-1);
|
||||
font-size: clamp(1.35rem, 3vw, 2rem);
|
||||
line-height: 1.2;
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin-bottom: var(--space-1);
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
margin-bottom: var(--space-1);
|
||||
color: var(--accent-strong);
|
||||
font-size: 0.78rem;
|
||||
font-weight: 800;
|
||||
letter-spacing: 0.14em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
|
||||
.lede {
|
||||
max-width: 65ch;
|
||||
margin-bottom: 0;
|
||||
color: var(--muted);
|
||||
font-size: 1.05rem;
|
||||
}
|
||||
|
||||
.connection-state {
|
||||
display: inline-flex;
|
||||
min-height: 2.75rem;
|
||||
align-items: center;
|
||||
gap: var(--space-2);
|
||||
padding: var(--space-2) var(--space-3);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
background: rgb(16 28 40 / 88%);
|
||||
color: var(--muted);
|
||||
font-weight: 700;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.status-dot {
|
||||
width: 0.7rem;
|
||||
height: 0.7rem;
|
||||
border: 2px solid currentColor;
|
||||
border-radius: 50%;
|
||||
background: currentColor;
|
||||
}
|
||||
|
||||
body[data-stream-state="connected"] .connection-state {
|
||||
color: var(--accent-strong);
|
||||
}
|
||||
|
||||
body[data-stream-state="reconnecting"] .connection-state,
|
||||
body[data-stream-state="polling"] .connection-state,
|
||||
body[data-stream-state="reset"] .connection-state {
|
||||
color: var(--warning);
|
||||
}
|
||||
|
||||
body[data-stream-state="error"] .connection-state {
|
||||
color: var(--danger);
|
||||
}
|
||||
|
||||
main {
|
||||
display: grid;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
|
||||
.filter-panel,
|
||||
.summary-panel,
|
||||
.results-panel {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
background: rgb(16 28 40 / 94%);
|
||||
box-shadow: 0 1rem 3rem rgb(0 0 0 / 18%);
|
||||
}
|
||||
|
||||
.filter-panel,
|
||||
.results-panel {
|
||||
padding: clamp(1rem, 4vw, 2rem);
|
||||
}
|
||||
|
||||
.section-heading {
|
||||
display: flex;
|
||||
align-items: start;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-4);
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
|
||||
.boundary-note,
|
||||
#results-summary {
|
||||
max-width: 34rem;
|
||||
margin-bottom: 0;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.filter-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: var(--space-3);
|
||||
}
|
||||
|
||||
label {
|
||||
display: grid;
|
||||
gap: 0.35rem;
|
||||
color: var(--muted);
|
||||
font-size: 0.9rem;
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
select {
|
||||
width: 100%;
|
||||
min-height: 2.75rem;
|
||||
padding: 0.6rem 2.4rem 0.6rem 0.75rem;
|
||||
border: 1px solid #587189;
|
||||
border-radius: 0.45rem;
|
||||
color: var(--text);
|
||||
background: #0b1722;
|
||||
}
|
||||
|
||||
select:disabled {
|
||||
color: #9fb0bd;
|
||||
border-style: dashed;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.button-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: var(--space-2);
|
||||
margin-top: var(--space-4);
|
||||
}
|
||||
|
||||
.button {
|
||||
min-height: 2.75rem;
|
||||
padding: 0.65rem 1rem;
|
||||
border: 1px solid #6a8298;
|
||||
border-radius: 0.45rem;
|
||||
color: var(--text);
|
||||
background: var(--surface-raised);
|
||||
cursor: pointer;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.button:hover {
|
||||
border-color: var(--accent-strong);
|
||||
background: #21384a;
|
||||
}
|
||||
|
||||
.button:active {
|
||||
transform: translateY(1px);
|
||||
}
|
||||
|
||||
.button.primary {
|
||||
color: var(--accent-ink);
|
||||
border-color: var(--accent);
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.button.primary:hover {
|
||||
background: var(--accent-strong);
|
||||
}
|
||||
|
||||
.form-error {
|
||||
margin: var(--space-3) 0 0;
|
||||
color: var(--danger);
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.summary-panel {
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.summary-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.summary-grid > div {
|
||||
min-width: 0;
|
||||
padding: var(--space-3) var(--space-4);
|
||||
border-right: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.summary-grid > div:last-child {
|
||||
border-right: 0;
|
||||
}
|
||||
|
||||
.summary-grid dt {
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.summary-grid dd {
|
||||
margin: 0.25rem 0 0;
|
||||
overflow-wrap: anywhere;
|
||||
font-size: 1.05rem;
|
||||
font-weight: 800;
|
||||
}
|
||||
|
||||
.results-heading {
|
||||
align-items: end;
|
||||
}
|
||||
|
||||
.session-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(min(100%, 19rem), 1fr));
|
||||
gap: var(--space-3);
|
||||
}
|
||||
|
||||
.session-card {
|
||||
min-width: 0;
|
||||
padding: var(--space-3);
|
||||
border: 1px solid #496177;
|
||||
border-radius: 0.6rem;
|
||||
background: #0b1722;
|
||||
}
|
||||
|
||||
.card-heading {
|
||||
display: flex;
|
||||
align-items: start;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-2);
|
||||
}
|
||||
|
||||
.card-heading h3 {
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.region-badge {
|
||||
flex: 0 0 auto;
|
||||
padding: 0.15rem 0.5rem;
|
||||
border: 1px solid #597187;
|
||||
border-radius: 999px;
|
||||
color: var(--muted);
|
||||
font-size: 0.75rem;
|
||||
font-weight: 750;
|
||||
}
|
||||
|
||||
.session-detail {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(5rem, auto) 1fr;
|
||||
gap: 0.25rem var(--space-2);
|
||||
margin: var(--space-3) 0 0;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.session-detail dt {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.session-detail dd {
|
||||
min-width: 0;
|
||||
margin: 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.capacity-row {
|
||||
margin-top: var(--space-3);
|
||||
}
|
||||
|
||||
.capacity-copy {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: var(--space-2);
|
||||
margin-bottom: 0.35rem;
|
||||
color: var(--muted);
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
meter {
|
||||
width: 100%;
|
||||
height: 0.65rem;
|
||||
accent-color: var(--accent);
|
||||
}
|
||||
|
||||
.metadata-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 0.35rem;
|
||||
margin: var(--space-3) 0 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.metadata-list li {
|
||||
max-width: 100%;
|
||||
padding: 0.2rem 0.45rem;
|
||||
overflow-wrap: anywhere;
|
||||
border-radius: 0.3rem;
|
||||
color: #d6e4ed;
|
||||
background: #1b2b39;
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.updated-age {
|
||||
margin: var(--space-3) 0 0;
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
padding: var(--space-6) var(--space-3);
|
||||
text-align: center;
|
||||
border: 1px dashed #587189;
|
||||
border-radius: 0.6rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.empty-state h3 {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
footer {
|
||||
padding-block: var(--space-5);
|
||||
color: var(--muted);
|
||||
font-size: 0.85rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.visually-hidden {
|
||||
position: absolute;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
padding: 0;
|
||||
overflow: hidden;
|
||||
clip: rect(0, 0, 0, 0);
|
||||
white-space: nowrap;
|
||||
border: 0;
|
||||
}
|
||||
|
||||
[hidden] {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
@media (max-width: 56rem) {
|
||||
.site-header,
|
||||
.section-heading {
|
||||
align-items: start;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.filter-grid,
|
||||
.summary-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.summary-grid > div:nth-child(2) {
|
||||
border-right: 0;
|
||||
}
|
||||
|
||||
.summary-grid > div:nth-child(-n + 2) {
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 36rem) {
|
||||
.site-header,
|
||||
main,
|
||||
footer {
|
||||
width: min(100% - 1rem, 80rem);
|
||||
}
|
||||
|
||||
.site-header {
|
||||
padding-block: var(--space-5) var(--space-4);
|
||||
}
|
||||
|
||||
.filter-grid,
|
||||
.summary-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.summary-grid > div {
|
||||
border-right: 0;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.summary-grid > div:last-child {
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
.button {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
scroll-behavior: auto !important;
|
||||
transition-duration: 0.01ms !important;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using System.Reflection;
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
|
||||
internal static class DiagnosticDashboardEndpoints
|
||||
{
|
||||
private const string ContentSecurityPolicy =
|
||||
"default-src 'none'; base-uri 'none'; connect-src 'self'; "
|
||||
+ "font-src 'self'; form-action 'none'; frame-ancestors 'none'; "
|
||||
+ "img-src 'self'; manifest-src 'none'; object-src 'none'; "
|
||||
+ "script-src 'self'; style-src 'self'";
|
||||
private static readonly byte[] Index = ReadAsset("index.html");
|
||||
private static readonly byte[] Script = ReadAsset("app.mjs");
|
||||
private static readonly byte[] Styles = ReadAsset("styles.css");
|
||||
|
||||
public static IEndpointRouteBuilder MapDiagnosticDashboardEndpoints(
|
||||
this IEndpointRouteBuilder endpoints)
|
||||
{
|
||||
RouteGroupBuilder dashboard = endpoints.MapGroup("/diagnostics")
|
||||
.ExcludeFromDescription();
|
||||
dashboard.MapGet("", ServeIndex);
|
||||
dashboard.MapGet("/app.mjs", ServeScript);
|
||||
dashboard.MapGet("/styles.css", ServeStyles);
|
||||
dashboard.MapGet("/config.json", ServeConfiguration);
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
private static IResult ServeIndex(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
|
||||
ServeAsset(context, configured.Value, Index, "text/html; charset=utf-8");
|
||||
|
||||
private static IResult ServeScript(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
|
||||
ServeAsset(context, configured.Value, Script, "text/javascript; charset=utf-8");
|
||||
|
||||
private static IResult ServeStyles(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured) =>
|
||||
ServeAsset(context, configured.Value, Styles, "text/css; charset=utf-8");
|
||||
|
||||
private static IResult ServeConfiguration(
|
||||
HttpContext context,
|
||||
[FromServices] IOptions<DiagnosticDashboardOptions> configured)
|
||||
{
|
||||
DiagnosticDashboardOptions options = configured.Value;
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
ApplySecurityHeaders(context.Response);
|
||||
return Results.Json(new
|
||||
{
|
||||
contractVersion = ContractLimits.ContractVersion,
|
||||
pollIntervalSeconds = options.PollIntervalSeconds,
|
||||
maximumRenderedSessions = options.MaximumRenderedSessions,
|
||||
scopes = options.Scopes.Select(static scope => new
|
||||
{
|
||||
gameId = scope.GameId,
|
||||
environmentId = scope.EnvironmentId,
|
||||
protocolVersions = scope.ProtocolVersions,
|
||||
regions = scope.Regions,
|
||||
visibility = "public",
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
private static IResult ServeAsset(
|
||||
HttpContext context,
|
||||
DiagnosticDashboardOptions options,
|
||||
byte[] content,
|
||||
string contentType)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
ApplySecurityHeaders(context.Response);
|
||||
return Results.Bytes(content, contentType);
|
||||
}
|
||||
|
||||
private static void ApplySecurityHeaders(HttpResponse response)
|
||||
{
|
||||
response.Headers.CacheControl = "no-store";
|
||||
response.Headers["Content-Security-Policy"] = ContentSecurityPolicy;
|
||||
response.Headers["X-Content-Type-Options"] = "nosniff";
|
||||
response.Headers["X-Frame-Options"] = "DENY";
|
||||
response.Headers["Cross-Origin-Opener-Policy"] = "same-origin";
|
||||
response.Headers["Cross-Origin-Resource-Policy"] = "same-origin";
|
||||
response.Headers["Permissions-Policy"] =
|
||||
"camera=(), geolocation=(), microphone=(), payment=(), usb=()";
|
||||
response.Headers["Referrer-Policy"] = "no-referrer";
|
||||
}
|
||||
|
||||
private static byte[] ReadAsset(string fileName)
|
||||
{
|
||||
Assembly assembly = typeof(DiagnosticDashboardEndpoints).Assembly;
|
||||
string resourceName = $"FinalFactory.Rendezvous.Server.Diagnostics.Assets.{fileName}";
|
||||
using Stream source = assembly.GetManifestResourceStream(resourceName)
|
||||
?? throw new InvalidOperationException($"Missing embedded dashboard asset {fileName}.");
|
||||
using MemoryStream destination = new();
|
||||
source.CopyTo(destination);
|
||||
return destination.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
using FinalFactory.Rendezvous.Contracts;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
|
||||
internal sealed record DiagnosticDashboardOptions
|
||||
{
|
||||
public const string SectionName = "Rendezvous:Diagnostics";
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
public int PollIntervalSeconds { get; init; } = 10;
|
||||
public int MaximumRenderedSessions { get; init; } = 100;
|
||||
public DiagnosticDashboardScope[] Scopes { get; init; } = [];
|
||||
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
List<string> errors = [];
|
||||
if (PollIntervalSeconds is < 5 or > 60)
|
||||
{
|
||||
errors.Add($"{SectionName}:PollIntervalSeconds must be between 5 and 60.");
|
||||
}
|
||||
if (MaximumRenderedSessions is < 10 or > 500)
|
||||
{
|
||||
errors.Add($"{SectionName}:MaximumRenderedSessions must be between 10 and 500.");
|
||||
}
|
||||
if (!Enabled)
|
||||
{
|
||||
return errors;
|
||||
}
|
||||
if (Scopes is null || Scopes.Length is < 1 or > 32)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes must contain between 1 and 32 allow-listed scopes when enabled.");
|
||||
return errors;
|
||||
}
|
||||
|
||||
HashSet<string> identities = new(StringComparer.Ordinal);
|
||||
foreach (DiagnosticDashboardScope? scope in Scopes)
|
||||
{
|
||||
if (scope is null)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes cannot contain null entries.");
|
||||
continue;
|
||||
}
|
||||
string gameId = scope.GameId ?? string.Empty;
|
||||
string environmentId = scope.EnvironmentId ?? string.Empty;
|
||||
uint[] protocolVersions = scope.ProtocolVersions ?? [];
|
||||
string[] regions = scope.Regions ?? [];
|
||||
if (!GameId.TryParse(gameId, out _)
|
||||
|| !EnvironmentId.TryParse(environmentId, out _))
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes contains an invalid game or environment identifier.");
|
||||
}
|
||||
if (protocolVersions.Length is < 1 or > 16
|
||||
|| protocolVersions.Any(static version => version == 0)
|
||||
|| protocolVersions.Distinct().Count() != protocolVersions.Length)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes protocol versions must contain 1-16 unique positive values.");
|
||||
}
|
||||
if (regions.Length is < 1 or > 16
|
||||
|| regions.Any(static region => !RegionId.TryParse(region ?? string.Empty, out _))
|
||||
|| regions.Distinct(StringComparer.Ordinal).Count() != regions.Length)
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes regions must contain 1-16 unique valid identifiers.");
|
||||
}
|
||||
|
||||
string identity = $"{gameId}\n{environmentId}";
|
||||
if (!identities.Add(identity))
|
||||
{
|
||||
errors.Add($"{SectionName}:Scopes contains a duplicate game/environment pair.");
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record DiagnosticDashboardScope
|
||||
{
|
||||
public string GameId { get; init; } = string.Empty;
|
||||
public string EnvironmentId { get; init; } = string.Empty;
|
||||
public uint[] ProtocolVersions { get; init; } = [];
|
||||
public string[] Regions { get; init; } = [];
|
||||
}
|
||||
@@ -53,6 +53,8 @@
|
||||
<Compile Include="ConnectionOutcomes/ConnectionOutcomeService.cs" />
|
||||
<Compile Include="Deployment/DeploymentOptions.cs" />
|
||||
<Compile Include="Deployment/GracefulDrainService.cs" />
|
||||
<Compile Include="Diagnostics/DiagnosticDashboardEndpoints.cs" />
|
||||
<Compile Include="Diagnostics/DiagnosticDashboardOptions.cs" />
|
||||
<Compile Include="Http/ContractEndpoints.cs" />
|
||||
<Compile Include="Http/RendezvousExceptionHandler.cs" />
|
||||
<Compile Include="JoinAttempts/JoinAttemptCursorCodec.cs" />
|
||||
@@ -60,6 +62,7 @@
|
||||
<Compile Include="Observability/AuditOptions.cs" />
|
||||
<Compile Include="Observability/AuditTrail.cs" />
|
||||
<Compile Include="Observability/HealthEndpoints.cs" />
|
||||
<Compile Include="Observability/PrometheusMetricsEndpoint.cs" />
|
||||
<Compile Include="Observability/RendezvousReadiness.cs" />
|
||||
<Compile Include="Observability/RendezvousTelemetry.cs" />
|
||||
<Compile Include="Observability/TelemetryMiddleware.cs" />
|
||||
@@ -88,4 +91,15 @@
|
||||
<Compile Include="Transport/UdpMediatorOptions.cs" />
|
||||
<Compile Include="Transport/UdpMediatorService.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<EmbeddedResource Include="Diagnostics/Assets/app.mjs">
|
||||
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.app.mjs</LogicalName>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="Diagnostics/Assets/index.html">
|
||||
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.index.html</LogicalName>
|
||||
</EmbeddedResource>
|
||||
<EmbeddedResource Include="Diagnostics/Assets/styles.css">
|
||||
<LogicalName>FinalFactory.Rendezvous.Server.Diagnostics.Assets.styles.css</LogicalName>
|
||||
</EmbeddedResource>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using FinalFactory.Rendezvous.Server.Provisioning;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Observability;
|
||||
|
||||
internal sealed record PrometheusMetricsOptions
|
||||
{
|
||||
public const string SectionName = "Rendezvous:Metrics";
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
public string BearerTokenSecretReference { get; init; } = string.Empty;
|
||||
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
if (!Enabled)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
string reference = BearerTokenSecretReference ?? string.Empty;
|
||||
bool environmentReference = reference.StartsWith("env:", StringComparison.Ordinal)
|
||||
&& reference.Length > "env:".Length;
|
||||
bool absoluteFileReference = reference.StartsWith("file:", StringComparison.Ordinal)
|
||||
&& Path.IsPathFullyQualified(reference["file:".Length..]);
|
||||
return reference.Length is < 5 or > 512
|
||||
|| !(environmentReference || absoluteFileReference)
|
||||
? [$"{SectionName}:BearerTokenSecretReference must be a bounded env: or absolute file: secret reference when metrics are enabled."]
|
||||
: [];
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class MetricsAccessCredential : IDisposable
|
||||
{
|
||||
private byte[]? _token;
|
||||
|
||||
private MetricsAccessCredential(byte[] token) => _token = token;
|
||||
|
||||
public static bool TryCreate(
|
||||
PrometheusMetricsOptions options,
|
||||
ISecretProvider secrets,
|
||||
out MetricsAccessCredential? credential)
|
||||
{
|
||||
credential = null;
|
||||
if (!options.Enabled
|
||||
|| !secrets.TryGetSecret(options.BearerTokenSecretReference, out SecretMaterial? material)
|
||||
|| material is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
using (material)
|
||||
{
|
||||
byte[] bytes = material.CopyBytes();
|
||||
int length = bytes.Length;
|
||||
while (length > 0 && bytes[length - 1] is (byte)'\r' or (byte)'\n')
|
||||
{
|
||||
length--;
|
||||
}
|
||||
bool valid = length is >= 32 and <= 128
|
||||
&& bytes.AsSpan(0, length).IndexOfAnyExceptInRange((byte)0x21, (byte)0x7e) < 0;
|
||||
if (!valid)
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(bytes);
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] token = bytes.AsSpan(0, length).ToArray();
|
||||
CryptographicOperations.ZeroMemory(bytes);
|
||||
credential = new(token);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Authorizes(HttpRequest request)
|
||||
{
|
||||
byte[]? expected = _token;
|
||||
string authorization = request.Headers.Authorization.ToString();
|
||||
const string prefix = "Bearer ";
|
||||
if (expected is null
|
||||
|| !authorization.StartsWith(prefix, StringComparison.Ordinal)
|
||||
|| authorization.Length - prefix.Length is < 32 or > 128)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
byte[] supplied = Encoding.UTF8.GetBytes(authorization[prefix.Length..]);
|
||||
try
|
||||
{
|
||||
return supplied.Length == expected.Length
|
||||
&& CryptographicOperations.FixedTimeEquals(supplied, expected);
|
||||
}
|
||||
finally
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(supplied);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
byte[]? token = Interlocked.Exchange(ref _token, null);
|
||||
if (token is not null)
|
||||
{
|
||||
CryptographicOperations.ZeroMemory(token);
|
||||
}
|
||||
}
|
||||
|
||||
public override string ToString() => "[MetricsAccessCredential: REDACTED]";
|
||||
}
|
||||
|
||||
internal static class PrometheusMetricsEndpoint
|
||||
{
|
||||
public static IEndpointRouteBuilder MapPrometheusMetricsEndpoint(
|
||||
this IEndpointRouteBuilder endpoints,
|
||||
PrometheusMetricsOptions options,
|
||||
MetricsAccessCredential? credential)
|
||||
{
|
||||
endpoints.MapGet("/metrics", (HttpContext context, RendezvousTelemetry telemetry) =>
|
||||
Export(context, telemetry, options, credential))
|
||||
.WithName("MetricsScrape")
|
||||
.ExcludeFromDescription();
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
private static IResult Export(
|
||||
HttpContext context,
|
||||
RendezvousTelemetry telemetry,
|
||||
PrometheusMetricsOptions options,
|
||||
MetricsAccessCredential? credential)
|
||||
{
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
if (credential is null || !credential.Authorizes(context.Request))
|
||||
{
|
||||
telemetry.RecordMetricsScrape("rejected");
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
telemetry.RecordMetricsScrape("accepted");
|
||||
context.Response.Headers.CacheControl = "no-store";
|
||||
context.Response.Headers["X-Content-Type-Options"] = "nosniff";
|
||||
return Results.Text(
|
||||
telemetry.RenderPrometheus(),
|
||||
"text/plain; version=0.0.4; charset=utf-8",
|
||||
Encoding.UTF8);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,10 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.Metrics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using FinalFactory.Rendezvous.Server.Browser;
|
||||
using FinalFactory.Rendezvous.Server.Provisioning;
|
||||
using FinalFactory.Rendezvous.Server.State;
|
||||
|
||||
namespace FinalFactory.Rendezvous.Server.Observability;
|
||||
@@ -10,6 +15,10 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
public const string ActivitySourceName = "FinalFactory.Rendezvous.Server";
|
||||
|
||||
private readonly InMemoryEphemeralRendezvousStore _store;
|
||||
private readonly SessionChangeJournal? _sessionChanges;
|
||||
private readonly ProvisioningRuntime? _provisioning;
|
||||
private readonly ConcurrentDictionary<MetricSeriesKey, long> _prometheusCounters = new();
|
||||
private readonly ConcurrentDictionary<MetricSeriesKey, PrometheusHistogram> _prometheusHistograms = new();
|
||||
private readonly Meter _meter = new(MeterName, "1.0.0");
|
||||
private readonly ActivitySource _activities = new(ActivitySourceName, "1.0.0");
|
||||
private readonly Counter<long> _httpRequests;
|
||||
@@ -22,9 +31,14 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
private readonly Counter<long> _operatorAuthentication;
|
||||
private readonly Histogram<double> _pairingLatency;
|
||||
|
||||
public RendezvousTelemetry(InMemoryEphemeralRendezvousStore store)
|
||||
public RendezvousTelemetry(
|
||||
InMemoryEphemeralRendezvousStore store,
|
||||
SessionChangeJournal? sessionChanges = null,
|
||||
ProvisioningRuntime? provisioning = null)
|
||||
{
|
||||
_store = store;
|
||||
_sessionChanges = sessionChanges;
|
||||
_provisioning = provisioning;
|
||||
_httpRequests = _meter.CreateCounter<long>("rendezvous.http.requests");
|
||||
_httpDuration = _meter.CreateHistogram<double>(
|
||||
"rendezvous.http.duration",
|
||||
@@ -75,9 +89,21 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
};
|
||||
_httpRequests.Add(1, tags);
|
||||
_httpDuration.Record(elapsedMilliseconds, tags);
|
||||
IncrementPrometheus(
|
||||
"rendezvous_http_requests_total",
|
||||
Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture))));
|
||||
ObservePrometheus(
|
||||
"rendezvous_http_duration_milliseconds",
|
||||
elapsedMilliseconds,
|
||||
Labels(("operation", operation), ("status_code", statusCode.ToString(CultureInfo.InvariantCulture))));
|
||||
}
|
||||
|
||||
public void RecordUdp(string operation, string result, double elapsedMilliseconds)
|
||||
public void RecordUdp(
|
||||
string operation,
|
||||
string result,
|
||||
double elapsedMilliseconds,
|
||||
int receivedBytes = 0,
|
||||
int responseBudgetBytes = 0)
|
||||
{
|
||||
TagList tags = new()
|
||||
{
|
||||
@@ -86,41 +112,315 @@ internal sealed class RendezvousTelemetry : IDisposable
|
||||
};
|
||||
_udpResults.Add(1, tags);
|
||||
_udpDuration.Record(elapsedMilliseconds, tags);
|
||||
IncrementPrometheus(
|
||||
"rendezvous_udp_results_total",
|
||||
Labels(("operation", operation), ("result", result)));
|
||||
ObservePrometheus(
|
||||
"rendezvous_udp_duration_milliseconds",
|
||||
elapsedMilliseconds,
|
||||
Labels(("operation", operation), ("result", result)));
|
||||
AddPrometheus(
|
||||
"rendezvous_udp_received_bytes_total",
|
||||
Math.Max(0, receivedBytes),
|
||||
Labels(("operation", operation)));
|
||||
AddPrometheus(
|
||||
"rendezvous_udp_response_budget_bytes_total",
|
||||
Math.Max(0, responseBudgetBytes),
|
||||
Labels(("operation", operation)));
|
||||
}
|
||||
|
||||
public void RecordLimiterDrop(string transport, string partition) =>
|
||||
public void RecordLimiterDrop(string transport, string partition)
|
||||
{
|
||||
_limiterDrops.Add(1, new TagList
|
||||
{
|
||||
{ "transport", transport },
|
||||
{ "partition", partition },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_limiter_drops_total",
|
||||
Labels(("transport", transport), ("partition", partition)));
|
||||
}
|
||||
|
||||
public void RecordAudit(string action, string result) =>
|
||||
public void RecordAudit(string action, string result)
|
||||
{
|
||||
_auditEvents.Add(1, new TagList
|
||||
{
|
||||
{ "action", action },
|
||||
{ "result", result },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_audit_events_total",
|
||||
Labels(("action", action), ("result", result)));
|
||||
}
|
||||
|
||||
public void RecordConnectionOutcome(string outcome, string elapsedBucket) =>
|
||||
public void RecordConnectionOutcome(string outcome, string elapsedBucket)
|
||||
{
|
||||
_connectionOutcomes.Add(1, new TagList
|
||||
{
|
||||
{ "outcome", outcome },
|
||||
{ "elapsed_bucket", elapsedBucket },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_connection_outcomes_total",
|
||||
Labels(("outcome", outcome), ("elapsed_bucket", elapsedBucket)));
|
||||
}
|
||||
|
||||
public void RecordOperatorAuthentication(string result) =>
|
||||
public void RecordOperatorAuthentication(string result)
|
||||
{
|
||||
_operatorAuthentication.Add(1, new TagList
|
||||
{
|
||||
{ "result", result },
|
||||
});
|
||||
IncrementPrometheus(
|
||||
"rendezvous_operator_authentication_total",
|
||||
Labels(("result", result)));
|
||||
}
|
||||
|
||||
public void RecordPairingLatency(double elapsedMilliseconds) =>
|
||||
public void RecordPairingLatency(double elapsedMilliseconds)
|
||||
{
|
||||
_pairingLatency.Record(elapsedMilliseconds);
|
||||
ObservePrometheus("rendezvous_pairing_latency_milliseconds", elapsedMilliseconds, string.Empty);
|
||||
}
|
||||
|
||||
public void RecordMetricsScrape(string result) => IncrementPrometheus(
|
||||
"rendezvous_metrics_scrapes_total",
|
||||
Labels(("result", result)));
|
||||
|
||||
public string RenderPrometheus()
|
||||
{
|
||||
StringBuilder output = new(16 * 1024);
|
||||
foreach (IGrouping<string, KeyValuePair<MetricSeriesKey, long>> family in _prometheusCounters
|
||||
.OrderBy(static item => item.Key.Name, StringComparer.Ordinal)
|
||||
.ThenBy(static item => item.Key.Labels, StringComparer.Ordinal)
|
||||
.GroupBy(static item => item.Key.Name, StringComparer.Ordinal))
|
||||
{
|
||||
output.Append("# TYPE ").Append(family.Key).Append(" counter\n");
|
||||
foreach (KeyValuePair<MetricSeriesKey, long> series in family)
|
||||
{
|
||||
AppendValue(output, series.Key.Name, series.Key.Labels, series.Value);
|
||||
}
|
||||
}
|
||||
foreach (IGrouping<string, KeyValuePair<MetricSeriesKey, PrometheusHistogram>> family in
|
||||
_prometheusHistograms
|
||||
.OrderBy(static item => item.Key.Name, StringComparer.Ordinal)
|
||||
.ThenBy(static item => item.Key.Labels, StringComparer.Ordinal)
|
||||
.GroupBy(static item => item.Key.Name, StringComparer.Ordinal))
|
||||
{
|
||||
output.Append("# TYPE ").Append(family.Key).Append(" histogram\n");
|
||||
foreach (KeyValuePair<MetricSeriesKey, PrometheusHistogram> series in family)
|
||||
{
|
||||
series.Value.Append(output, series.Key.Name, series.Key.Labels);
|
||||
}
|
||||
}
|
||||
|
||||
EphemeralStoreSnapshot store = _store.GetMetricsSnapshot();
|
||||
AppendGauge(output, "rendezvous_store_active_listings", store.ActiveListings);
|
||||
AppendGauge(output, "rendezvous_store_fresh_presence_bindings", store.FreshPresenceBindings);
|
||||
AppendGauge(
|
||||
output,
|
||||
"rendezvous_store_awaiting_presence_listings",
|
||||
Math.Max(0, store.ActiveListings - store.FreshPresenceBindings));
|
||||
AppendGauge(output, "rendezvous_store_active_leases", store.ActiveListings);
|
||||
AppendGauge(output, "rendezvous_store_active_attempts", store.ActiveJoinAttempts);
|
||||
AppendGauge(output, "rendezvous_queue_depth", store.ActiveJoinAttempts);
|
||||
AppendGauge(output, "rendezvous_store_replay_markers", store.ReplayMarkers);
|
||||
AppendGauge(output, "rendezvous_store_available", store.IsAvailable ? 1 : 0);
|
||||
AppendGauge(output, "rendezvous_store_draining", store.IsDraining ? 1 : 0);
|
||||
AppendCounter(output, "rendezvous_store_expiry_churn_total", store.ExpiryChurn);
|
||||
if (_sessionChanges is not null)
|
||||
{
|
||||
AppendGauge(output, "rendezvous_browser_sse_subscribers", _sessionChanges.SubscriberCount);
|
||||
AppendGauge(output, "rendezvous_browser_sse_tenants", _sessionChanges.SubscribedTenantCount);
|
||||
AppendGauge(output, "rendezvous_browser_replay_entries", _sessionChanges.ReplayCount);
|
||||
}
|
||||
AppendProcessMetrics(output);
|
||||
AppendSigningKeyMetrics(output);
|
||||
return output.ToString();
|
||||
}
|
||||
|
||||
private void AppendSigningKeyMetrics(StringBuilder output)
|
||||
{
|
||||
if (_provisioning is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
DateTimeOffset now = DateTimeOffset.UtcNow;
|
||||
SigningKeyStatus[] statuses = _provisioning.SigningKeys.GetStatuses(now).ToArray();
|
||||
output.Append("# TYPE rendezvous_signing_keys gauge\n");
|
||||
foreach (IGrouping<string, SigningKeyStatus> state in statuses.GroupBy(
|
||||
static status => status.Status,
|
||||
StringComparer.Ordinal))
|
||||
{
|
||||
AppendValue(
|
||||
output,
|
||||
"rendezvous_signing_keys",
|
||||
Labels(("state", state.Key)),
|
||||
state.Count());
|
||||
}
|
||||
double seconds = statuses
|
||||
.Where(static status => status.Status == "signing")
|
||||
.Select(status => Math.Max(0, (status.SignUntil - now).TotalSeconds))
|
||||
.DefaultIfEmpty(0)
|
||||
.Min();
|
||||
AppendGauge(output, "rendezvous_signing_key_sign_seconds_remaining", seconds);
|
||||
}
|
||||
|
||||
private static void AppendProcessMetrics(StringBuilder output)
|
||||
{
|
||||
using Process process = Process.GetCurrentProcess();
|
||||
process.Refresh();
|
||||
AppendCounter(output, "process_cpu_seconds_total", process.TotalProcessorTime.TotalSeconds);
|
||||
AppendGauge(output, "process_resident_memory_bytes", process.WorkingSet64);
|
||||
AppendGauge(output, "process_virtual_memory_bytes", process.VirtualMemorySize64);
|
||||
AppendGauge(output, "process_threads", process.Threads.Count);
|
||||
try
|
||||
{
|
||||
AppendGauge(
|
||||
output,
|
||||
"process_open_file_descriptors",
|
||||
Directory.EnumerateFileSystemEntries("/proc/self/fd").Count());
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException
|
||||
or UnauthorizedAccessException)
|
||||
{
|
||||
}
|
||||
AppendGauge(output, "dotnet_gc_heap_size_bytes", GC.GetTotalMemory(forceFullCollection: false));
|
||||
output.Append("# TYPE dotnet_gc_collections_total counter\n");
|
||||
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "0")), GC.CollectionCount(0));
|
||||
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "1")), GC.CollectionCount(1));
|
||||
AppendValue(output, "dotnet_gc_collections_total", Labels(("generation", "2")), GC.CollectionCount(2));
|
||||
AppendGauge(output, "dotnet_thread_pool_threads", ThreadPool.ThreadCount);
|
||||
ThreadPool.GetAvailableThreads(out int workerThreads, out int completionThreads);
|
||||
AppendGauge(output, "dotnet_thread_pool_available_worker_threads", workerThreads);
|
||||
AppendGauge(output, "dotnet_thread_pool_available_completion_threads", completionThreads);
|
||||
}
|
||||
|
||||
private void IncrementPrometheus(string name, string labels) =>
|
||||
_prometheusCounters.AddOrUpdate(new(name, labels), 1, static (_, current) => current + 1);
|
||||
|
||||
private void AddPrometheus(string name, long value, string labels)
|
||||
{
|
||||
if (value <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
_prometheusCounters.AddOrUpdate(new(name, labels), value, (_, current) => current + value);
|
||||
}
|
||||
|
||||
private void ObservePrometheus(string name, double value, string labels) =>
|
||||
_prometheusHistograms.GetOrAdd(new(name, labels), static _ => new()).Observe(value);
|
||||
|
||||
private static string Labels(params (string Name, string Value)[] labels)
|
||||
{
|
||||
if (labels.Length == 0)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
return "{" + string.Join(',', labels.Select(static label =>
|
||||
$"{label.Name}=\"{EscapeLabel(NormalizeLabel(label.Value))}\"")) + "}";
|
||||
}
|
||||
|
||||
private static string NormalizeLabel(string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value) || value.Length > 64)
|
||||
{
|
||||
return "other";
|
||||
}
|
||||
return value.All(static character => char.IsAsciiLetterOrDigit(character)
|
||||
|| character is '-' or '_' or '.')
|
||||
? value
|
||||
: "other";
|
||||
}
|
||||
|
||||
private static string EscapeLabel(string value) => value
|
||||
.Replace("\\", "\\\\", StringComparison.Ordinal)
|
||||
.Replace("\"", "\\\"", StringComparison.Ordinal)
|
||||
.Replace("\n", "\\n", StringComparison.Ordinal);
|
||||
|
||||
private static void AppendCounter(
|
||||
StringBuilder output,
|
||||
string name,
|
||||
double value,
|
||||
string labels = "")
|
||||
{
|
||||
output.Append("# TYPE ").Append(name).Append(" counter\n");
|
||||
AppendValue(output, name, labels, value);
|
||||
}
|
||||
|
||||
private static void AppendGauge(
|
||||
StringBuilder output,
|
||||
string name,
|
||||
double value,
|
||||
string labels = "")
|
||||
{
|
||||
output.Append("# TYPE ").Append(name).Append(" gauge\n");
|
||||
AppendValue(output, name, labels, value);
|
||||
}
|
||||
|
||||
private static void AppendValue(StringBuilder output, string name, string labels, double value) =>
|
||||
output.Append(name)
|
||||
.Append(labels)
|
||||
.Append(' ')
|
||||
.Append(value.ToString("R", CultureInfo.InvariantCulture))
|
||||
.Append('\n');
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_activities.Dispose();
|
||||
_meter.Dispose();
|
||||
}
|
||||
|
||||
private readonly record struct MetricSeriesKey(string Name, string Labels);
|
||||
|
||||
private sealed class PrometheusHistogram
|
||||
{
|
||||
private static readonly double[] Bounds = [1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000];
|
||||
private readonly object _gate = new();
|
||||
private readonly long[] _buckets = new long[Bounds.Length];
|
||||
private long _count;
|
||||
private double _sum;
|
||||
|
||||
public void Observe(double value)
|
||||
{
|
||||
if (!double.IsFinite(value) || value < 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
lock (_gate)
|
||||
{
|
||||
_count++;
|
||||
_sum += value;
|
||||
for (int index = 0; index < Bounds.Length; index++)
|
||||
{
|
||||
if (value <= Bounds[index])
|
||||
{
|
||||
_buckets[index]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Append(StringBuilder output, string name, string labels)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
for (int index = 0; index < Bounds.Length; index++)
|
||||
{
|
||||
AppendValue(
|
||||
output,
|
||||
name + "_bucket",
|
||||
AddLabel(labels, "le", Bounds[index].ToString("R", CultureInfo.InvariantCulture)),
|
||||
_buckets[index]);
|
||||
}
|
||||
AppendValue(output, name + "_bucket", AddLabel(labels, "le", "+Inf"), _count);
|
||||
AppendValue(output, name + "_sum", labels, _sum);
|
||||
AppendValue(output, name + "_count", labels, _count);
|
||||
}
|
||||
}
|
||||
|
||||
private static string AddLabel(string labels, string name, string value) =>
|
||||
string.IsNullOrEmpty(labels)
|
||||
? $"{{{name}=\"{value}\"}}"
|
||||
: labels[..^1] + $",{name}=\"{value}\"}}";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ using FinalFactory.Rendezvous.Server.Abuse;
|
||||
using FinalFactory.Rendezvous.Server.Browser;
|
||||
using FinalFactory.Rendezvous.Server.ConnectionOutcomes;
|
||||
using FinalFactory.Rendezvous.Server.Deployment;
|
||||
using FinalFactory.Rendezvous.Server.Diagnostics;
|
||||
using FinalFactory.Rendezvous.Server.Http;
|
||||
using FinalFactory.Rendezvous.Server.JoinAttempts;
|
||||
using FinalFactory.Rendezvous.Server.Observability;
|
||||
@@ -251,6 +252,37 @@ if (!builder.Environment.IsDevelopment() && !isOpenApiGeneration)
|
||||
}
|
||||
|
||||
builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(deploymentOptions));
|
||||
DiagnosticDashboardOptions diagnosticDashboardOptions = builder.Configuration
|
||||
.GetSection(DiagnosticDashboardOptions.SectionName)
|
||||
.Get<DiagnosticDashboardOptions>() ?? new DiagnosticDashboardOptions();
|
||||
IReadOnlyList<string> diagnosticErrors = diagnosticDashboardOptions.Validate();
|
||||
if (diagnosticErrors.Count > 0)
|
||||
{
|
||||
throw new DeploymentConfigurationException(diagnosticErrors);
|
||||
}
|
||||
builder.Services.AddSingleton(
|
||||
Microsoft.Extensions.Options.Options.Create(diagnosticDashboardOptions));
|
||||
PrometheusMetricsOptions metricsOptions = builder.Configuration
|
||||
.GetSection(PrometheusMetricsOptions.SectionName)
|
||||
.Get<PrometheusMetricsOptions>() ?? new PrometheusMetricsOptions();
|
||||
IReadOnlyList<string> metricsErrors = metricsOptions.Validate();
|
||||
if (metricsErrors.Count > 0)
|
||||
{
|
||||
throw new DeploymentConfigurationException(metricsErrors);
|
||||
}
|
||||
builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(metricsOptions));
|
||||
MetricsAccessCredential? metricsCredential = null;
|
||||
if (metricsOptions.Enabled && !isOpenApiGeneration)
|
||||
{
|
||||
EnvironmentSecretProvider metricsSecrets = new();
|
||||
if (!MetricsAccessCredential.TryCreate(metricsOptions, metricsSecrets, out metricsCredential)
|
||||
|| metricsCredential is null)
|
||||
{
|
||||
throw new DeploymentConfigurationException(
|
||||
[$"{PrometheusMetricsOptions.SectionName}:BearerTokenSecretReference did not resolve to 32-128 visible ASCII bytes."]);
|
||||
}
|
||||
builder.Services.AddSingleton(metricsCredential);
|
||||
}
|
||||
builder.Services.Configure<HostOptions>(options =>
|
||||
options.ShutdownTimeout = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds + 10));
|
||||
|
||||
@@ -349,6 +381,8 @@ app.MapOpenApi();
|
||||
app.MapRendezvousContractEndpoints();
|
||||
app.MapOperatorEndpoints();
|
||||
app.MapRendezvousHealthEndpoints();
|
||||
app.MapDiagnosticDashboardEndpoints();
|
||||
app.MapPrometheusMetricsEndpoint(metricsOptions, metricsCredential);
|
||||
|
||||
await app.RunAsync();
|
||||
|
||||
|
||||
@@ -82,7 +82,9 @@ internal sealed class NatMediationProcessor(
|
||||
telemetry?.RecordUdp(
|
||||
"frozen",
|
||||
result.ToString(),
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds);
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds,
|
||||
encoded.Length,
|
||||
result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -157,7 +159,7 @@ internal sealed class NatMediationProcessor(
|
||||
observedPublicEndpoint,
|
||||
token,
|
||||
introductionSink,
|
||||
cancellationToken);
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
internal NatMediationResult ProcessRequestAfterIngress(
|
||||
@@ -165,6 +167,7 @@ internal sealed class NatMediationProcessor(
|
||||
IPEndPoint observedPublicEndpoint,
|
||||
string token,
|
||||
INatIntroductionSink introductionSink,
|
||||
int receivedBytes = 0,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
long started = Stopwatch.GetTimestamp();
|
||||
@@ -178,7 +181,9 @@ internal sealed class NatMediationProcessor(
|
||||
telemetry?.RecordUdp(
|
||||
"litenet",
|
||||
result.ToString(),
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds);
|
||||
Stopwatch.GetElapsedTime(started).TotalMilliseconds,
|
||||
receivedBytes,
|
||||
result == NatMediationResult.Introduced ? 2 * ContractLimits.UdpDatagramMaxBytes : 0);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ internal sealed class UdpMediatorService : BackgroundService
|
||||
&& token is not null)
|
||||
{
|
||||
_ = processor.ProcessRequestAfterIngress(
|
||||
claimedLocalEndpoint, endPoint, token, sink);
|
||||
claimedLocalEndpoint, endPoint, token, sink, length);
|
||||
}
|
||||
|
||||
// Every inbound packet is consumed here. NatPunchModule is used only for outbound introductions.
|
||||
|
||||
Reference in New Issue
Block a user