feat(deployment): add secure Linux runtime (#17)

2026-07-16 15:03:04 +02:00
parent be732de7c9
commit 08729ae25c
23 changed files with 1941 additions and 28 deletions
@@ -0,0 +1,235 @@
+using System.ComponentModel.DataAnnotations;
+using System.Net;
+using FinalFactory.Rendezvous.Server.Abuse;
+
+namespace FinalFactory.Rendezvous.Server.Deployment;
+
+internal sealed record DeploymentOptions
+{
+    public const string SectionName = "Rendezvous:Deployment";
+
+    [Required]
+    public string PublicHttpBaseUrl { get; init; } = string.Empty;
+
+    [Required]
+    public string PublicUdpHost { get; init; } = string.Empty;
+
+    [Range(1, 65_535)]
+    public int PublicUdpPort { get; init; } = 9050;
+
+    [Range(1, 30)]
+    public int DrainDeadlineSeconds { get; init; } = 30;
+
+    [Range(0, 5)]
+    public int MinimumDrainSeconds { get; init; } = 1;
+
+    public bool SingleActiveInstance { get; init; } = true;
+
+    public bool AllowPrivatePublicEndpoints { get; init; }
+
+    public IReadOnlyList<string> ValidateProduction(
+        AbuseProtectionOptions abuseProtection,
+        string? allowedHosts)
+    {
+        List<string> errors = [];
+        if (!SingleActiveInstance)
+        {
+            errors.Add("Rendezvous:Deployment:SingleActiveInstance must be true because ephemeral state is not shared between replicas.");
+        }
+
+        if (MinimumDrainSeconds >= DrainDeadlineSeconds)
+        {
+            errors.Add("Rendezvous:Deployment:MinimumDrainSeconds must be less than DrainDeadlineSeconds.");
+        }
+
+        if (DrainDeadlineSeconds is < 1 or > 30)
+        {
+            errors.Add("Rendezvous:Deployment:DrainDeadlineSeconds must be between 1 and 30.");
+        }
+
+        if (MinimumDrainSeconds is < 0 or > 5)
+        {
+            errors.Add("Rendezvous:Deployment:MinimumDrainSeconds must be between 0 and 5.");
+        }
+
+        if (PublicUdpPort is < 1 or > 65_535)
+        {
+            errors.Add("Rendezvous:Deployment:PublicUdpPort must be between 1 and 65535.");
+        }
+
+        ValidateHttpEndpoint(errors);
+        ValidateUdpEndpoint(errors);
+        ValidateAllowedHosts(errors, allowedHosts, PublicHttpBaseUrl);
+
+        if (abuseProtection.TrustedProxyAddresses is not { Length: > 0 })
+        {
+            errors.Add(
+                "Rendezvous:AbuseProtection:TrustedProxyAddresses must list the exact TLS proxy addresses; forwarded headers are rejected without this trust boundary.");
+        }
+
+        return errors;
+    }
+
+    private void ValidateHttpEndpoint(List<string> errors)
+    {
+        if (!Uri.TryCreate(PublicHttpBaseUrl, UriKind.Absolute, out Uri? endpoint)
+            || !string.Equals(endpoint.Scheme, Uri.UriSchemeHttps, StringComparison.Ordinal)
+            || !string.IsNullOrEmpty(endpoint.UserInfo)
+            || !string.IsNullOrEmpty(endpoint.Query)
+            || !string.IsNullOrEmpty(endpoint.Fragment)
+            || endpoint.AbsolutePath != "/")
+        {
+            errors.Add(
+                "Rendezvous:Deployment:PublicHttpBaseUrl must be an absolute HTTPS origin with no credentials, path, query, or fragment (for example, https://rendezvous.your-company.tld/).");
+            return;
+        }
+
+        if (!AllowPrivatePublicEndpoints && !IsPublicHost(endpoint.Host))
+        {
+            errors.Add(
+                "Rendezvous:Deployment:PublicHttpBaseUrl must use a public DNS name or address; set AllowPrivatePublicEndpoints=true only for an isolated deployment smoke test.");
+        }
+    }
+
+    private void ValidateUdpEndpoint(List<string> errors)
+    {
+        if (string.IsNullOrWhiteSpace(PublicUdpHost)
+            || PublicUdpHost.Contains("//", StringComparison.Ordinal)
+            || PublicUdpHost.Contains(':', StringComparison.Ordinal) && !IPAddress.TryParse(PublicUdpHost, out _)
+            || Uri.CheckHostName(PublicUdpHost) == UriHostNameType.Unknown)
+        {
+            errors.Add(
+                "Rendezvous:Deployment:PublicUdpHost must contain only the advertised DNS name or IP address; configure the port separately.");
+            return;
+        }
+
+        if (!AllowPrivatePublicEndpoints && !IsPublicHost(PublicUdpHost))
+        {
+            errors.Add(
+                "Rendezvous:Deployment:PublicUdpHost must use a public DNS name or address; set AllowPrivatePublicEndpoints=true only for an isolated deployment smoke test.");
+        }
+    }
+
+    private static void ValidateAllowedHosts(
+        List<string> errors,
+        string? allowedHosts,
+        string publicHttpBaseUrl)
+    {
+        string[] hosts = (allowedHosts ?? string.Empty).Split(
+            ';',
+            StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+        if (hosts.Length == 0 || hosts.Any(static host => host is "*" or "+"))
+        {
+            errors.Add(
+                "AllowedHosts must explicitly list the public HTTP host in production; wildcard or empty host filtering is unsafe.");
+            return;
+        }
+
+        if (Uri.TryCreate(publicHttpBaseUrl, UriKind.Absolute, out Uri? endpoint)
+            && !hosts.Contains(endpoint.Host, StringComparer.OrdinalIgnoreCase))
+        {
+            errors.Add(
+                "AllowedHosts must contain the exact host advertised by Rendezvous:Deployment:PublicHttpBaseUrl.");
+        }
+    }
+
+    private static bool IsPublicHost(string host)
+    {
+        if (!IPAddress.TryParse(host, out IPAddress? address))
+        {
+            return Uri.CheckHostName(host) == UriHostNameType.Dns
+                && host.Contains('.', StringComparison.Ordinal)
+                && !IsReservedDnsName(host);
+        }
+
+        return IsGloballyRoutableUnicast(address);
+    }
+
+    private static bool IsReservedDnsName(string host)
+    {
+        string normalized = host.TrimEnd('.');
+        string[] reservedSuffixes =
+        [
+            "localhost",
+            "local",
+            "invalid",
+            "test",
+            "example",
+            "example.com",
+            "example.net",
+            "example.org",
+            "home.arpa",
+            "alt",
+            "onion",
+        ];
+        return reservedSuffixes.Any(suffix =>
+            string.Equals(normalized, suffix, StringComparison.OrdinalIgnoreCase)
+            || normalized.EndsWith($".{suffix}", StringComparison.OrdinalIgnoreCase));
+    }
+
+    private static bool IsGloballyRoutableUnicast(IPAddress address)
+    {
+        if (address.IsIPv4MappedToIPv6)
+        {
+            address = address.MapToIPv4();
+        }
+
+        byte[] bytes = address.GetAddressBytes();
+        if (address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork)
+        {
+            return !(bytes[0] is 0 or 10 or 127
+                || bytes[0] == 100 && bytes[1] is >= 64 and <= 127
+                || bytes[0] == 169 && bytes[1] == 254
+                || bytes[0] == 172 && bytes[1] is >= 16 and <= 31
+                || bytes[0] == 192
+                    && (bytes[1] == 0 && bytes[2] is 0 or 2
+                        || bytes[1] == 88 && bytes[2] == 99
+                        || bytes[1] == 168)
+                || bytes[0] == 198
+                    && (bytes[1] is 18 or 19
+                        || bytes[1] == 51 && bytes[2] == 100)
+                || bytes[0] == 203 && bytes[1] == 0 && bytes[2] == 113
+                || bytes[0] >= 224);
+        }
+
+        return address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6
+            && !IPAddress.IsLoopback(address)
+            && !address.Equals(IPAddress.IPv6Any)
+            && !address.IsIPv6LinkLocal
+            && !address.IsIPv6SiteLocal
+            && !address.IsIPv6Multicast
+            && (bytes[0] & 0xe0) == 0x20
+            && !HasPrefix(bytes, [0x20, 0x01, 0x00], 23)
+            && !HasPrefix(bytes, [0x20, 0x01, 0x0d, 0xb8], 32)
+            && !HasPrefix(bytes, [0x20, 0x02], 16)
+            && !HasPrefix(bytes, [0x3f, 0xff, 0x00], 20);
+    }
+
+    private static bool HasPrefix(byte[] address, byte[] prefix, int bitCount)
+    {
+        int fullBytes = bitCount / 8;
+        for (int index = 0; index < fullBytes; index++)
+        {
+            if (address[index] != prefix[index])
+            {
+                return false;
+            }
+        }
+
+        int remainingBits = bitCount % 8;
+        if (remainingBits == 0)
+        {
+            return true;
+        }
+
+        int mask = 0xff << (8 - remainingBits);
+        return (address[fullBytes] & mask) == (prefix[fullBytes] & mask);
+    }
+}
+
+internal sealed class DeploymentConfigurationException(IReadOnlyList<string> errors)
+    : InvalidOperationException(
+        "Production deployment configuration is invalid:" + Environment.NewLine
+        + string.Join(Environment.NewLine, errors.Select(static error => $"- {error}")))
+{
+}
@@ -0,0 +1,98 @@
+using System.Diagnostics;
+using FinalFactory.Rendezvous.Server.State;
+using Microsoft.Extensions.Options;
+
+namespace FinalFactory.Rendezvous.Server.Deployment;
+
+internal sealed partial class GracefulDrainService : IHostedService, IDisposable
+{
+    private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(50);
+    private readonly InMemoryEphemeralRendezvousStore _store;
+    private readonly IHostApplicationLifetime _lifetime;
+    private readonly DeploymentOptions _options;
+    private readonly ILogger<GracefulDrainService> _logger;
+    private readonly object _gate = new();
+    private CancellationTokenRegistration _stoppingRegistration;
+    private Task? _drainTask;
+
+    public GracefulDrainService(
+        InMemoryEphemeralRendezvousStore store,
+        IHostApplicationLifetime lifetime,
+        IOptions<DeploymentOptions> options,
+        ILogger<GracefulDrainService> logger)
+    {
+        _store = store;
+        _lifetime = lifetime;
+        _options = options.Value;
+        _logger = logger;
+    }
+
+    public Task StartAsync(CancellationToken cancellationToken)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+        _stoppingRegistration = _lifetime.ApplicationStopping.Register(
+            () => EnsureDrainAsync().GetAwaiter().GetResult());
+        return Task.CompletedTask;
+    }
+
+    public Task StopAsync(CancellationToken cancellationToken)
+    {
+        // ApplicationStopping callbacks run before hosted services and listeners
+        // stop. StopAsync is the idempotent fallback for directly driven hosts.
+        _ = cancellationToken;
+        return EnsureDrainAsync();
+    }
+
+    public void Dispose() => _stoppingRegistration.Dispose();
+
+    private Task EnsureDrainAsync()
+    {
+        lock (_gate)
+        {
+            return _drainTask ??= DrainAsync();
+        }
+    }
+
+    private async Task DrainAsync()
+    {
+        _store.BeginDrain(CancellationToken.None);
+        TimeSpan deadline = TimeSpan.FromSeconds(_options.DrainDeadlineSeconds);
+        TimeSpan minimum = TimeSpan.FromSeconds(_options.MinimumDrainSeconds);
+        long startedAt = Stopwatch.GetTimestamp();
+        LogDrainStarted(_logger, _options.DrainDeadlineSeconds);
+        try
+        {
+            while (Stopwatch.GetElapsedTime(startedAt) < deadline)
+            {
+                TimeSpan elapsed = Stopwatch.GetElapsedTime(startedAt);
+                if (elapsed >= minimum && _store.GetActiveJoinAttemptCountForDrain() == 0)
+                {
+                    break;
+                }
+
+                TimeSpan remaining = deadline - elapsed;
+                await Task.Delay(
+                    remaining < PollInterval ? remaining : PollInterval,
+                    CancellationToken.None).ConfigureAwait(false);
+            }
+        }
+        finally
+        {
+            _store.MarkUnavailable();
+            double elapsedMilliseconds = Stopwatch.GetElapsedTime(startedAt).TotalMilliseconds;
+            LogDrainFinished(_logger, elapsedMilliseconds);
+        }
+    }
+
+    [LoggerMessage(
+        EventId = 1,
+        Level = LogLevel.Information,
+        Message = "Graceful drain started with a {DrainDeadlineSeconds}-second deadline")]
+    private static partial void LogDrainStarted(ILogger logger, int drainDeadlineSeconds);
+
+    [LoggerMessage(
+        EventId = 2,
+        Level = LogLevel.Information,
+        Message = "Graceful drain finished after {ElapsedMilliseconds:F0} ms; ephemeral state was cleared")]
+    private static partial void LogDrainFinished(ILogger logger, double elapsedMilliseconds);
+}
@@ -3,6 +3,7 @@ using FinalFactory.Rendezvous.Contracts;
 using FinalFactory.Rendezvous.Server.Abuse;
 using FinalFactory.Rendezvous.Server.Browser;
 using FinalFactory.Rendezvous.Server.ConnectionOutcomes;
+using FinalFactory.Rendezvous.Server.Deployment;
 using FinalFactory.Rendezvous.Server.Http;
 using FinalFactory.Rendezvous.Server.JoinAttempts;
 using FinalFactory.Rendezvous.Server.Observability;
@@ -235,8 +236,29 @@ AbuseProtectionOptions configuredAbuseProtection = builder.Configuration
 builder.Services.Configure<ForwardedHeadersOptions>(options =>
    TrustedProxyForwarding.Configure(options, configuredAbuseProtection));

+DeploymentOptions deploymentOptions = builder.Configuration
+    .GetSection(DeploymentOptions.SectionName)
+    .Get<DeploymentOptions>() ?? new DeploymentOptions();
+if (!builder.Environment.IsDevelopment() && !isOpenApiGeneration)
+{
+    IReadOnlyList<string> deploymentErrors = deploymentOptions.ValidateProduction(
+        configuredAbuseProtection,
+        builder.Configuration["AllowedHosts"]);
+    if (deploymentErrors.Count > 0)
+    {
+        throw new DeploymentConfigurationException(deploymentErrors);
+    }
+}
+
+builder.Services.AddSingleton(Microsoft.Extensions.Options.Options.Create(deploymentOptions));
+builder.Services.Configure<HostOptions>(options =>
+    options.ShutdownTimeout = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds + 10));
+
 SystemRendezvousClock rendezvousClock = new();
-EphemeralStoreOptions stateOptions = new();
+EphemeralStoreOptions stateOptions = new()
+{
+    GracefulDrainLifetime = TimeSpan.FromSeconds(deploymentOptions.DrainDeadlineSeconds),
+};
 InMemoryEphemeralRendezvousStore stateStore = new(
    stateOptions,
    rendezvousClock,
@@ -304,10 +326,12 @@ if (!isOpenApiGeneration)
    builder.Services.AddSingleton<NatMediationProcessor>();
    builder.Services.AddHostedService(static services =>
        services.GetRequiredService<UdpMediatorService>());
+    // Hosted services stop in reverse registration order. Drain must complete while
+    // Kestrel and the UDP mediator are still able to finish bounded in-flight work.
+    builder.Services.AddHostedService<GracefulDrainService>();
 }

 WebApplication app = builder.Build();
-app.Lifetime.ApplicationStopping.Register(() => stateStore.BeginDrain());

 if (TrustedProxyForwarding.IsEnabled(configuredAbuseProtection))
 {
@@ -40,18 +40,32 @@ internal sealed class SecretMaterial : IDisposable

 internal sealed class EnvironmentSecretProvider : ISecretProvider
 {
-    private const string Prefix = "env:";
+    private const string EnvironmentPrefix = "env:";
+    private const string FilePrefix = "file:";
+    private const int MaximumSecretBytes = 4096;

    public bool TryGetSecret(string reference, out SecretMaterial? secret)
    {
        secret = null;
-        if (!reference.StartsWith(Prefix, StringComparison.Ordinal)
-            || reference.Length == Prefix.Length)
+        if (reference.StartsWith(EnvironmentPrefix, StringComparison.Ordinal)
+            && reference.Length > EnvironmentPrefix.Length)
        {
-            return false;
+            return TryGetEnvironmentSecret(reference[EnvironmentPrefix.Length..], out secret);
        }

-        string? encoded = Environment.GetEnvironmentVariable(reference[Prefix.Length..]);
+        if (reference.StartsWith(FilePrefix, StringComparison.Ordinal)
+            && reference.Length > FilePrefix.Length)
+        {
+            return TryGetFileSecret(reference[FilePrefix.Length..], out secret);
+        }
+
+        return false;
+    }
+
+    private static bool TryGetEnvironmentSecret(string variableName, out SecretMaterial? secret)
+    {
+        secret = null;
+        string? encoded = Environment.GetEnvironmentVariable(variableName);
        if (string.IsNullOrEmpty(encoded))
        {
            return false;
@@ -60,6 +74,12 @@ internal sealed class EnvironmentSecretProvider : ISecretProvider
        try
        {
            byte[] bytes = Convert.FromBase64String(encoded);
+            if (bytes.Length is 0 or > MaximumSecretBytes)
+            {
+                CryptographicOperations.ZeroMemory(bytes);
+                return false;
+            }
+
            secret = new SecretMaterial(bytes);
            CryptographicOperations.ZeroMemory(bytes);
            return true;
@@ -69,6 +89,47 @@ internal sealed class EnvironmentSecretProvider : ISecretProvider
            return false;
        }
    }
+
+    private static bool TryGetFileSecret(string path, out SecretMaterial? secret)
+    {
+        secret = null;
+        byte[]? bytes = null;
+        try
+        {
+            FileInfo file = new(path);
+            if (!file.Exists
+                || !Path.IsPathFullyQualified(path)
+                || file.LinkTarget is not null
+                || file.Length is <= 0 or > MaximumSecretBytes)
+            {
+                return false;
+            }
+
+            bytes = File.ReadAllBytes(path);
+            if (bytes.Length is 0 or > MaximumSecretBytes)
+            {
+                return false;
+            }
+
+            secret = new SecretMaterial(bytes);
+            return true;
+        }
+        catch (Exception exception) when (exception is IOException
+                                           or UnauthorizedAccessException
+                                           or ArgumentException
+                                           or NotSupportedException
+                                           or System.Security.SecurityException)
+        {
+            return false;
+        }
+        finally
+        {
+            if (bytes is not null)
+            {
+                CryptographicOperations.ZeroMemory(bytes);
+            }
+        }
+    }
 }

 internal sealed class EphemeralDevelopmentSecretProvider : ISecretProvider, IDisposable
@@ -15,6 +15,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
    private readonly Dictionary<MediationHandle, SessionListingId> _presenceHandles = [];
    private readonly Dictionary<MediationHandle, PresenceEntry> _presence = [];
    private readonly Dictionary<JoinAttemptId, AttemptEntry> _attempts = [];
+    private readonly PriorityQueue<AttemptExpiry, long> _attemptExpiries = new();
    private readonly Dictionary<JoinAttemptId, OutcomeReportEntry> _outcomeReports = [];
    private readonly Dictionary<MediationHandle, JoinAttemptId> _attemptHandles = [];
    private readonly Dictionary<string, IdempotencyEntry> _idempotency = new(StringComparer.Ordinal);
@@ -82,6 +83,15 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
        }
    }

+    internal int GetActiveJoinAttemptCountForDrain()
+    {
+        lock (_gate)
+        {
+            _expiryChurn += RemoveExpiredAttempts(_monotonicClock.Elapsed);
+            return _attempts.Count;
+        }
+    }
+
    internal EphemeralStoreSnapshot GetMetricsSnapshot()
    {
        lock (_gate)
@@ -451,6 +461,9 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
            now + _options.JoinAttemptLifetime,
            WallDeadline(now, _options.JoinAttemptLifetime));
        _attempts.Add(command.AttemptId, attempt);
+        _attemptExpiries.Enqueue(
+            new AttemptExpiry(command.AttemptId, attempt.Deadline),
+            attempt.Deadline.Ticks);
        _outcomeReports.Add(command.AttemptId, new(
            command.ListingId,
            command.ClientSubject,
@@ -920,15 +933,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
            _presence.Remove(handle);
        }

-        JoinAttemptId[] expiredAttempts = _attempts
-            .Where(item => item.Value.Deadline <= now)
-            .Select(static item => item.Key)
-            .ToArray();
-        _expiryChurn += expiredAttempts.Length;
-        foreach (JoinAttemptId attemptId in expiredAttempts)
-        {
-            RemoveAttempt(attemptId);
-        }
+        _expiryChurn += RemoveExpiredAttempts(now);

        JoinAttemptId[] expiredOutcomes = _outcomeReports
            .Where(item => item.Value.Deadline <= now)
@@ -958,6 +963,7 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
        _presenceHandles.Clear();
        _presence.Clear();
        _attempts.Clear();
+        _attemptExpiries.Clear();
        _outcomeReports.Clear();
        _attemptHandles.Clear();
        _idempotency.Clear();
@@ -1000,6 +1006,24 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
        }
    }

+    private int RemoveExpiredAttempts(TimeSpan now)
+    {
+        int removed = 0;
+        while (_attemptExpiries.TryPeek(out AttemptExpiry candidate, out long deadlineTicks)
+               && deadlineTicks <= now.Ticks)
+        {
+            _attemptExpiries.Dequeue();
+            if (_attempts.TryGetValue(candidate.AttemptId, out AttemptEntry? current)
+                && current.Deadline == candidate.Deadline)
+            {
+                RemoveAttempt(candidate.AttemptId);
+                removed++;
+            }
+        }
+
+        return removed;
+    }
+
    private bool HandleExists(MediationHandle handle) =>
        _presenceHandles.ContainsKey(handle) || _attemptHandles.ContainsKey(handle);

@@ -1195,6 +1219,8 @@ internal sealed class InMemoryEphemeralRendezvousStore : IEphemeralRendezvousSto
        public bool IsCancelled { get; set; }
    }

+    private readonly record struct AttemptExpiry(JoinAttemptId AttemptId, TimeSpan Deadline);
+
    private sealed class OutcomeReportEntry(
        SessionListingId listingId,
        string clientSubject,
@@ -1,5 +1,14 @@
 {
  "Rendezvous": {
+    "Deployment": {
+      "PublicHttpBaseUrl": "",
+      "PublicUdpHost": "",
+      "PublicUdpPort": 9050,
+      "DrainDeadlineSeconds": 30,
+      "MinimumDrainSeconds": 1,
+      "SingleActiveInstance": true,
+      "AllowPrivatePublicEndpoints": false
+    },
    "Udp": {
      "ListenAddress": "0.0.0.0",
      "Port": 9050,