feat(operations): add production readiness gate (#23)

2026-07-16 22:19:51 +02:00
parent 6bad659c12
commit 00d5ff7764
13 changed files with 782 additions and 6 deletions
@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+PROJECT="$ROOT/src/FinalFactory.Rendezvous.TestClient/FinalFactory.Rendezvous.TestClient.csproj"
+ROLE="${RENDEZVOUS_CANARY_ROLE:-}"
+TOPOLOGY="${RENDEZVOUS_CANARY_TOPOLOGY:-}"
+ADDRESS_FAMILY="${RENDEZVOUS_CANARY_ADDRESS_FAMILY:-ipv4}"
+SERVICE_URL="${RENDEZVOUS_CANARY_HTTP_URL:-}"
+MEDIATOR="${RENDEZVOUS_CANARY_UDP_ENDPOINT:-}"
+GAME_ID="${RENDEZVOUS_CANARY_GAME_ID:-space-game}"
+ENVIRONMENT_ID="${RENDEZVOUS_CANARY_ENVIRONMENT_ID:-production-canary}"
+REGION="${RENDEZVOUS_CANARY_REGION:-production-canary}"
+PROTOCOL_VERSION="${RENDEZVOUS_CANARY_PROTOCOL_VERSION:-1}"
+TIMEOUT_SECONDS="${RENDEZVOUS_CANARY_TIMEOUT_SECONDS:-60}"
+RUN_SECONDS="${RENDEZVOUS_CANARY_RUN_SECONDS:-900}"
+OUTPUT="${RENDEZVOUS_CANARY_OUTPUT:-$ROOT/artifacts/canary/${ROLE:-unknown}-${TOPOLOGY:-unknown}.json}"
+COORDINATION_FILE="${RENDEZVOUS_CANARY_COORDINATION_FILE:-}"
+LISTING_ID="${RENDEZVOUS_CANARY_LISTING_ID:-}"
+REQUIRE_CLEAN="${RENDEZVOUS_CANARY_REQUIRE_CLEAN:-true}"
+KEEP_RAW="${RENDEZVOUS_CANARY_KEEP_RAW:-false}"
+
+usage() {
+    printf '%s\n' \
+        'Set RENDEZVOUS_CANARY_ROLE to host, client-success, or client-expected-failure.' \
+        'Also set RENDEZVOUS_CANARY_TOPOLOGY, RENDEZVOUS_CANARY_HTTP_URL, and' \
+        'RENDEZVOUS_CANARY_UDP_ENDPOINT. See docs/operations/production-readiness.md.' >&2
+    exit 2
+}
+
+for command in date dotnet git jq mktemp; do
+    command -v "$command" >/dev/null || {
+        printf 'Missing required command: %s\n' "$command" >&2
+        exit 2
+    }
+done
+
+case "$ROLE" in
+    host|client-success|client-expected-failure) ;;
+    *) usage ;;
+esac
+case "$TOPOLOGY" in
+    same-lan|home-nat|firewall-blocked-udp|restrictive-cgnat|ipv6-direct) ;;
+    *) usage ;;
+esac
+case "$ADDRESS_FAMILY" in
+    ipv4|ipv6) ;;
+    *) printf 'RENDEZVOUS_CANARY_ADDRESS_FAMILY must be ipv4 or ipv6.\n' >&2; exit 2 ;;
+esac
+if [[ "$TOPOLOGY" == ipv6-direct && "$ADDRESS_FAMILY" != ipv6 ]]; then
+    printf 'The ipv6-direct topology requires RENDEZVOUS_CANARY_ADDRESS_FAMILY=ipv6.\n' >&2
+    exit 2
+fi
+if [[ "$TOPOLOGY" =~ ^(firewall-blocked-udp|restrictive-cgnat)$ \
+    && "$ROLE" == client-success ]]; then
+    printf 'Failure topologies must use the client-expected-failure role.\n' >&2
+    exit 2
+fi
+if [[ -z "$SERVICE_URL" || -z "$MEDIATOR" ]]; then
+    usage
+fi
+if [[ ! "$TIMEOUT_SECONDS" =~ ^[0-9]+$ ]] \
+    || (( TIMEOUT_SECONDS < 1 || TIMEOUT_SECONDS > 300 )); then
+    printf 'RENDEZVOUS_CANARY_TIMEOUT_SECONDS must be an integer from 1 through 300.\n' >&2
+    exit 2
+fi
+if [[ ! "$RUN_SECONDS" =~ ^[0-9]+$ ]] \
+    || (( RUN_SECONDS < 60 || RUN_SECONDS > 3600 )); then
+    printf 'RENDEZVOUS_CANARY_RUN_SECONDS must be an integer from 60 through 3600.\n' >&2
+    exit 2
+fi
+if [[ ! "$PROTOCOL_VERSION" =~ ^[0-9]+$ ]] || (( PROTOCOL_VERSION < 1 )); then
+    printf 'RENDEZVOUS_CANARY_PROTOCOL_VERSION must be a positive integer.\n' >&2
+    exit 2
+fi
+if [[ "$REQUIRE_CLEAN" != true && "$REQUIRE_CLEAN" != false ]]; then
+    printf 'RENDEZVOUS_CANARY_REQUIRE_CLEAN must be true or false.\n' >&2
+    exit 2
+fi
+if [[ "$KEEP_RAW" != true && "$KEEP_RAW" != false ]]; then
+    printf 'RENDEZVOUS_CANARY_KEEP_RAW must be true or false.\n' >&2
+    exit 2
+fi
+
+cd "$ROOT"
+commit="$(git rev-parse HEAD)"
+tree_state=clean
+if [[ -n "$(git status --porcelain)" ]]; then
+    tree_state=dirty
+fi
+if [[ "$REQUIRE_CLEAN" == true && "$tree_state" != clean ]]; then
+    printf 'Formal canary evidence requires a clean source tree.\n' >&2
+    exit 2
+fi
+
+if [[ "$ROLE" == host ]]; then
+    if [[ -z "$COORDINATION_FILE" ]]; then
+        printf 'The host role requires RENDEZVOUS_CANARY_COORDINATION_FILE.\n' >&2
+        exit 2
+    fi
+    if [[ -z "${RENDEZVOUS_PUBLISHER_CREDENTIAL:-}" ]]; then
+        printf 'The host role requires RENDEZVOUS_PUBLISHER_CREDENTIAL.\n' >&2
+        exit 2
+    fi
+else
+    if [[ ! "$LISTING_ID" =~ ^[0-9a-fA-F-]{36}$ ]]; then
+        printf 'A client role requires a UUID in RENDEZVOUS_CANARY_LISTING_ID.\n' >&2
+        exit 2
+    fi
+fi
+
+umask 077
+raw_dir="$(mktemp -d "${TMPDIR:-/tmp}/rendezvous-canary.XXXXXXXX")"
+raw_log="$raw_dir/events.jsonl"
+run_succeeded=false
+host_pid=''
+cleanup() {
+    local status="$?"
+    if [[ -n "$host_pid" ]] && kill -0 "$host_pid" 2>/dev/null; then
+        kill -TERM "$host_pid" 2>/dev/null || true
+        wait "$host_pid" 2>/dev/null || true
+    fi
+    if [[ "$run_succeeded" == true && "$KEEP_RAW" == false ]]; then
+        rm -rf "$raw_dir"
+    else
+        printf 'Private raw canary events retained at %s\n' "$raw_dir" >&2
+    fi
+    return "$status"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+common_arguments=(
+    --service "$SERVICE_URL"
+    --mediator "$MEDIATOR"
+    --game "$GAME_ID"
+    --environment "$ENVIRONMENT_ID"
+    --region "$REGION"
+    --protocol "$PROTOCOL_VERSION"
+    --script
+    --json
+    --timeout-seconds "$TIMEOUT_SECONDS"
+)
+
+exit_code=0
+if [[ "$ROLE" == host ]]; then
+    dotnet run --project "$PROJECT" --configuration Release --no-build -- \
+        host "${common_arguments[@]}" --exit-after-echo --run-seconds "$RUN_SECONDS" \
+        >"$raw_log" 2>&1 &
+    host_pid="$!"
+    ready=false
+    for ((iteration = 0; iteration < TIMEOUT_SECONDS * 4; iteration++)); do
+        if jq -e 'select(.event == "host.ready" and .status == "ready")' "$raw_log" \
+            >/dev/null 2>&1; then
+            ready=true
+            break
+        fi
+        if ! kill -0 "$host_pid" 2>/dev/null; then
+            break
+        fi
+        sleep 0.25
+    done
+    if [[ "$ready" != true ]]; then
+        printf 'The canary host did not become ready within the bounded startup window.\n' >&2
+        kill -TERM "$host_pid" 2>/dev/null || true
+        wait "$host_pid" 2>/dev/null || true
+        exit 1
+    fi
+    observed_listing="$(jq -r 'select(.event == "host.registered") | .listingId' "$raw_log" | tail -n 1)"
+    if [[ ! "$observed_listing" =~ ^[0-9a-f-]{36}$ ]]; then
+        printf 'The canary host did not produce a valid coordination identifier.\n' >&2
+        kill -TERM "$host_pid" 2>/dev/null || true
+        wait "$host_pid" 2>/dev/null || true
+        exit 1
+    fi
+    coordination_parent="$(dirname "$COORDINATION_FILE")"
+    mkdir -p "$coordination_parent"
+    coordination_temp="$COORDINATION_FILE.tmp.$$"
+    printf '%s\n' "$observed_listing" >"$coordination_temp"
+    chmod 600 "$coordination_temp"
+    mv "$coordination_temp" "$COORDINATION_FILE"
+    printf 'Host ready; securely transfer the private coordination file to the client operator.\n'
+    set +e
+    wait "$host_pid"
+    exit_code="$?"
+    set -e
+elif [[ "$ROLE" == client-success ]]; then
+    set +e
+    dotnet run --project "$PROJECT" --configuration Release --no-build -- \
+        join "${common_arguments[@]}" --listing "$LISTING_ID" >"$raw_log" 2>&1
+    exit_code="$?"
+    set -e
+else
+    set +e
+    dotnet run --project "$PROJECT" --configuration Release --no-build -- \
+        join "${common_arguments[@]}" --listing "$LISTING_ID" >"$raw_log" 2>&1
+    exit_code="$?"
+    set -e
+fi
+
+checks='{}'
+if [[ "$ROLE" == host ]]; then
+    [[ "$exit_code" -eq 0 ]]
+    jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "host.direct-traffic" and .status == "verified" and .addressFamily == $family)' "$raw_log" >/dev/null
+    jq -e 'select(.event == "host.deregistered" and .status == "complete")' "$raw_log" >/dev/null
+    checks='{"authenticatedDirectTraffic":true,"deregistered":true}'
+elif [[ "$ROLE" == client-success ]]; then
+    [[ "$exit_code" -eq 0 ]]
+    jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "join.connected" and .status == "connected" and .addressFamily == $family)' "$raw_log" >/dev/null
+    jq -e --arg family "$ADDRESS_FAMILY" 'select(.event == "join.direct-traffic" and .status == "verified" and .addressFamily == $family)' "$raw_log" >/dev/null
+    jq -e 'select(.event == "join.outcome-report" and .status == "accepted")' "$raw_log" >/dev/null
+    checks='{"authenticatedDirectTraffic":true,"typedOutcomeReported":true}'
+else
+    [[ "$exit_code" -eq 12 ]]
+    jq -e 'select((.event == "join.traversal" or .event == "join.authorization") and .status == "failed" and (.outcome | type == "string") and (.outcome | length > 0))' "$raw_log" >/dev/null
+    jq -e 'select(.event == "join.fallback" and (.status == "available" or .status == "unavailable") and (.outcome | type == "string") and (.outcome | length > 0))' "$raw_log" >/dev/null
+    checks='{"boundedTypedFailure":true,"fallbackPolicyReported":true}'
+fi
+
+mkdir -p "$(dirname "$OUTPUT")"
+jq -n \
+    --arg commit "$commit" \
+    --arg treeState "$tree_state" \
+    --arg timestampUtc "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    --arg role "$ROLE" \
+    --arg topology "$TOPOLOGY" \
+    --arg addressFamily "$ADDRESS_FAMILY" \
+    --argjson checks "$checks" \
+    '{schemaVersion:1,kind:"rendezvous-real-network-canary",commit:$commit,treeState:$treeState,timestampUtc:$timestampUtc,role:$role,topology:$topology,addressFamily:$addressFamily,result:"pass",checks:$checks,dataRetention:{rawEvents:"deleted-after-success",identifiers:"not-in-summary",networkEndpoints:"not-in-summary"}}' \
+    >"$OUTPUT"
+
+run_succeeded=true
+printf 'Real-network canary passed; sanitized evidence: %s\n' "$OUTPUT"