using System.Data.Common; using MeterVault.Core.Analysis; using MeterVault.Core.Analysis.Coverage; using MeterVault.Core.Analysis.Totals; using MeterVault.Core.Analysis.Virtual; using MeterVault.Core.Domain; using MeterVault.Infrastructure.Ingestion; using MeterVault.Infrastructure.Persistence; using Microsoft.EntityFrameworkCore; namespace MeterVault.Infrastructure.Analysis; /// /// One analysis request from catalog to result: which series and meters it needs, what to read for them, and how the /// read data becomes bucket values (D-14, D-27), totals (D-22), comparisons (D-07) and availability (D-19). /// /// /// The order matters and is fixed: targets → physical leaves → coverage → bucket plan and comparison → matched /// coverage → every window a sum will need is registered → the rollups and edge rows are loaded, once per table → /// values are computed from memory. Registering before loading is what keeps the reader to one query per table while /// still summing arbitrary ranges (a matched range can end mid-day). /// internal sealed class AnalysisRun { private static readonly BucketValue PendingValue = new(null, BucketStatus.Pending, Provenance.None, ValueIssue.AnalysisPending); private readonly MeterVaultDbContext _db; private readonly DbConnection _connection; private readonly AnalysisCatalog _catalog; private readonly AnalysisRequest _request; private readonly ResolvedPeriod _period; private readonly TimeZoneInfo _zone; /// Every physical meter whose coverage is read (availability needs the whole scope). private readonly Dictionary _leaves = []; /// The physical meters whose rollups are read (series, members and virtual sources). private readonly HashSet _dataLeaves = []; private readonly Dictionary> _leavesOf = []; private readonly Dictionary _freshness = []; private readonly Dictionary _afterNow = []; private readonly List _problems = []; private List _seriesIds = []; private List<(int EnergyTypeId, MeasureGroup Group)> _measures = []; private Side _current = null!; private Side? _comparison; private ComparisonResolution? _resolution; private ResolvedPeriod? _comparisonPeriod; private IReadOnlyList _pairs = []; private readonly Dictionary _matched = []; private bool _readsToday; public AnalysisRun(MeterVaultDbContext db, AnalysisCatalog catalog, AnalysisRequest request, TimeZoneInfo zone) { _db = db; _connection = db.Database.GetDbConnection(); _catalog = catalog; _request = request; _period = request.Period; _zone = zone; } private DateTimeOffset Now => _period.Now; /// The physical meters this run read, with what it registered and loaded for each (for tests of the D-15 budget). internal IReadOnlyDictionary Leaves => _leaves; public async Task ExecuteAsync(BucketPlan? plan, CancellationToken cancellationToken) { ChooseTargets(); CollectLeaves(includeScope: true); await LoadCoverageAsync(cancellationToken).ConfigureAwait(false); plan ??= BucketPlanner.Plan(_period, BucketSize.Auto, CoarsestNeeded(), _request.MaxPoints); if (plan.Refused) { return new AnalysisResult(_request, plan, [], [], ScopeAvailability.None, []) { Refusal = AnalysisRefusal.TooManyPoints }; } foreach (var id in _dataLeaves) { _leaves[id].UseMonths = plan.Size is BucketSize.Month or BucketSize.Year && !_leaves[id].IsVirtualSource; } _current = new Side(plan.Buckets, PeriodBucket.Of(_period), Now); await ResolveComparisonAsync(plan, cancellationToken).ConfigureAwait(false); MatchCoverage(); RegisterWindows(); await LoadDataAsync(cancellationToken).ConfigureAwait(false); if (!_request.QuantitiesOnly) { await LoadFreshnessAsync(cancellationToken).ConfigureAwait(false); } foreach (var id in _dataLeaves) { if (AfterNowOf(_leaves[id]) is { } block) { _afterNow[id] = block; } } var series = _seriesIds.Select(MeterSeries).ToList(); var measures = _measures.Select(m => MeasureSeries(m.EnergyTypeId, m.Group)).ToList(); var availability = await AvailabilityAsync(cancellationToken).ConfigureAwait(false); CollectProblems(); return new AnalysisResult(_request, plan, series, measures, availability, Deduplicated(_problems)) { Comparison = _resolution is null ? null : new AnalysisComparison(_resolution, _comparisonPeriod, _pairs), Classification = Classification(), }; } /// /// The buckets chooses for the request (D-05) from the coverage of the data it reads — /// the plan would make, without reading any rollups. /// public async Task PlanAsync(CancellationToken cancellationToken) { ChooseTargets(); CollectLeaves(includeScope: false); await LoadCoverageAsync(cancellationToken).ConfigureAwait(false); return BucketPlanner.Plan(_period, BucketSize.Auto, CoarsestNeeded(), _request.MaxPoints); } public async Task AvailabilityOnlyAsync(CancellationToken cancellationToken) { CollectLeaves(includeScope: true); await LoadCoverageAsync(cancellationToken).ConfigureAwait(false); return await AvailabilityAsync(cancellationToken).ConfigureAwait(false); } // ---------------------------------------------------------------- targets and leaves /// The meter series and measure totals the scope asks for. private void ChooseTargets() { var scope = _request.Scope; switch (scope.Kind) { case AnalysisScopeKind.Meters: foreach (var id in scope.MeterIds) { if (_catalog.Find(id) is null) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.UnknownMeter, id)); } else { _seriesIds.Add(id); } } break; case AnalysisScopeKind.EnergyType: var type = scope.EnergyTypeId!.Value; _measures = [.. _catalog.Totals.ForType(type).Measures.Select(g => (type, g))]; if (_request.IncludeMeterSeries) { _seriesIds = [.. ScopeMeters().OrderBy(id => id)]; } break; default: _measures = [.. _catalog.Totals.Types.OrderBy(t => t.Key).SelectMany(t => t.Value.Measures.Select(g => (t.Key, g)))]; if (_request.IncludeMeterSeries) { _seriesIds = [.. ScopeMeters().OrderBy(id => id)]; } break; } } /// The meters of the scope: the selection, the energy type's meters, or every meter. private IEnumerable ScopeMeters() => _request.Scope.Kind switch { AnalysisScopeKind.Meters => _request.Scope.MeterIds.Where(id => _catalog.Find(id) is not null), AnalysisScopeKind.EnergyType => _catalog.Meters.Values.Where(m => m.EnergyTypeId == _request.Scope.EnergyTypeId).Select(m => m.Id), _ => _catalog.Meters.Keys, }; /// The meters whose cost the scope is about (D-19): the selection, or the billed meters of the type(s). private IEnumerable CostMeters() => _request.Scope.Kind switch { AnalysisScopeKind.Meters => ScopeMeters(), AnalysisScopeKind.EnergyType => _catalog.Totals.ForType(_request.Scope.EnergyTypeId!.Value).Billing.Items, _ => _catalog.Totals.BillItems, }; /// /// Expands the targets to the physical meters they read, through evaluable virtual definitions only. Data leaves /// feed values; the scope's and the bill's other meters only need their coverage, for availability. /// private void CollectLeaves(bool includeScope) { var dataTargets = _seriesIds.Concat(_measures.SelectMany(m => m.Group.MeterIds)).Distinct().ToList(); foreach (var id in dataTargets) { foreach (var leaf in LeavesOf(id)) { _dataLeaves.Add(leaf); Leaf(leaf); } MarkVirtualSources(id, []); } if (!includeScope) { return; } foreach (var id in ScopeMeters().Concat(CostMeters()).Distinct()) { foreach (var leaf in LeavesOf(id)) { Leaf(leaf); } } } /// The physical meters a meter reads: itself, or the leaves of its evaluable formula; none when not evaluable. private IReadOnlyList LeavesOf(int meterId) { if (_leavesOf.TryGetValue(meterId, out var cached)) { return cached; } var leaves = new SortedSet(); var seen = new HashSet(); var pending = new Stack([meterId]); while (pending.Count > 0) { var id = pending.Pop(); if (!seen.Add(id) || _catalog.Find(id) is not { } meter) { continue; } if (!meter.IsVirtual) { leaves.Add(id); continue; } foreach (var source in meter.Formula?.MeterIds ?? []) { pending.Push(source); } } IReadOnlyList result = [.. leaves]; _leavesOf[meterId] = result; return result; } /// Marks the physical meters a virtual meter reads directly or through nested ones: they are read day by day. private void MarkVirtualSources(int meterId, HashSet seen) { if (!seen.Add(meterId) || _catalog.Find(meterId) is not { IsVirtual: true, Formula: { } formula }) { return; } foreach (var source in formula.MeterIds) { if (_catalog.Find(source) is { IsVirtual: false }) { Leaf(source).IsVirtualSource = true; } else { MarkVirtualSources(source, seen); } } } private LeafData Leaf(int meterId) { if (!_leaves.TryGetValue(meterId, out var leaf)) { _leaves[meterId] = leaf = new LeafData(_catalog.Meters[meterId], _zone); } return leaf; } private async Task LoadCoverageAsync(CancellationToken cancellationToken) { var runs = await AnalysisQueries.CoverageAsync(_connection, _leaves.Keys, cancellationToken).ConfigureAwait(false); foreach (var (id, leaf) in _leaves) { leaf.Runs = runs.GetValueOrDefault(id) ?? []; } } /// /// The coarsest resolution among the data covering the period (D-05): auto never plans finer. What each run asks /// for is — a run divided at month boundaries resolves /// months whatever its class (A-03), and a run coarser than a month only pushes the chart to years when years /// can place it (A-41). /// private ResolutionClass? CoarsestNeeded() => ResolutionClassifier.PlanningResolution( _dataLeaves .SelectMany(id => _leaves[id].CappedRuns(Now)) .Where(run => !run.IsGap && run.From < _period.To && run.To > _period.From), _zone); // ---------------------------------------------------------------- comparison and matched coverage private async Task ResolveComparisonAsync(BucketPlan plan, CancellationToken cancellationToken) { if (_request.Comparison.Kind == ComparisonKind.None) { return; } _resolution = ComparisonResolver.Resolve(_period, _request.Comparison); if (!_resolution.IsApplicable) { return; } var comparison = _resolution.Period; _comparisonPeriod = comparison.ToResolvedPeriod(_period); _pairs = ComparisonResolver.PairBuckets(_period, comparison, plan.Buckets); _comparison = new Side([.. _pairs.Select(p => p.Comparison)], PeriodBucket.Of(_comparisonPeriod), _comparisonPeriod.Now); var stamps = await AnalysisQueries.OpeningBalancesAsync(_connection, _dataLeaves, cancellationToken).ConfigureAwait(false); foreach (var (id, stamp) in stamps) { _leaves[id].OpeningBalanceStamp = stamp; } } /// The coverage both periods share (D-07), per series: the meter's own, or all its sources jointly. private void MatchCoverage() { if (_comparison is null || _resolution?.Period is not { } comparison) { return; } foreach (var id in _seriesIds) { var meter = _catalog.Meters[id]; var leaves = LeavesOf(id).Select(l => _leaves[l]).ToList(); var own = !meter.IsVirtual && !_request.AsContributors; _matched[MeterKey(id)] = Match(comparison, leaves, own); } foreach (var (type, group) in _measures) { var leaves = group.MeterIds.SelectMany(LeavesOf).Distinct().Select(l => _leaves[l]).ToList(); _matched[MeasureKey(type, group)] = Match(comparison, leaves, own: false); } } private MatchedCoverageResult Match(ComparisonPeriod comparison, List leaves, bool own) { if (leaves.Count == 0 || leaves.Exists(l => l.Meter.IsPending)) { return MatchedCoverageResult.NotComparable; } IReadOnlyList> sources = [.. leaves.Select(l => own ? l.Runs : l.ServiceRuns)]; IReadOnlyList balances = [.. leaves.Where(l => l.OpeningBalanceStamp is not null).Select(l => l.OpeningBalanceStamp!.Value)]; var current = MatchSide.OfSources(_period.From, _period.To, Now, _zone, sources, balances); var previous = MatchSide.OfSources(comparison.From, comparison.To, _comparison!.Cutoff, _zone, sources, balances); return MatchedCoverage.Match(current, previous, instant => comparison.MapInstant(instant, _zone)); } // ---------------------------------------------------------------- registering and loading /// Registers every window a sum will read, so the loads fetch exactly those rows. private void RegisterWindows() { List sides = [_current]; if (_comparison is not null) { sides.Add(_comparison); } foreach (var id in _dataLeaves) { var leaf = _leaves[id]; foreach (var side in sides) { foreach (var bucket in side.Buckets) { leaf.Request(bucket.From, bucket.To); } if (leaf.IsVirtualSource) { foreach (var window in side.DayWindows(_zone)) { leaf.Request(window.From, window.To); } } } } // Matched pieces are only summed: their partial days are summed in the database, not loaded (D-15). foreach (var (key, matched) in _matched) { foreach (var leaf in LeavesOfKey(key)) { foreach (var piece in matched.Pieces) { leaf.RequestSummed(piece.Current.From, piece.Current.To, _current.Cutoff); leaf.RequestSummed(piece.Comparison.From, piece.Comparison.To, _comparison!.Cutoff); } } } // Rows of today that close after now go to the "recorded after now" block (D-04). var today = RangeParts.LocalDate(Now, _zone); _readsToday = ReachesNow() && _period.FirstDay <= today && (_period.Preset == PeriodPreset.AllHistory || _period.NominalLastDay() >= today); if (_readsToday) { foreach (var id in _dataLeaves) { _leaves[id].RequestRawDay(today); } } } private IEnumerable LeavesOfKey(string key) { var ids = key.StartsWith('m') ? LeavesOf(_seriesIds.First(id => MeterKey(id) == key)) : _measures.Where(m => MeasureKey(m.EnergyTypeId, m.Group) == key).SelectMany(m => m.Group.MeterIds).SelectMany(LeavesOf).Distinct(); return ids.Select(id => _leaves[id]); } /// True when the requested range reaches now or lies after it: then rows may close after now (D-04). private bool ReachesNow() => _period.IsToDate || _period.ExtendsPastNow || _period.HasNotStarted(); private async Task LoadDataAsync(CancellationToken cancellationToken) { var data = _dataLeaves.ToDictionary(id => id, id => _leaves[id]); var today = RangeParts.LocalDate(Now, _zone); var afterFrom = _period.FirstDay > today ? _period.FirstDay : today.AddDays(1); DateOnly? afterTo = _period.Preset == PeriodPreset.AllHistory ? null : _period.NominalLastDay().AddDays(1); IReadOnlyCollection afterIds = ReachesNow() ? _dataLeaves : []; await AnalysisQueries.DayRollupsAsync(_connection, data, afterIds, afterFrom, afterTo, Now, cancellationToken).ConfigureAwait(false); await AnalysisQueries.MonthRollupsAsync(_connection, data, cancellationToken).ConfigureAwait(false); await AnalysisQueries.RawDaysAsync(_connection, data, _zone, cancellationToken).ConfigureAwait(false); await AnalysisQueries.WindowSumsAsync(_connection, data, cancellationToken).ConfigureAwait(false); foreach (var leaf in data.Values) { leaf.Seal(); } } /// Freshness of every data leaf (D-18): the last reading and event, and the live sources' rhythm. /// /// The mark — the meter's last reading — is read from its stored rollup state, which the recompute behind every /// write path records (A-40); an import-only meter's mark is years old and must stay exact, so it is never /// guessed from a bounded sample. Only the *rhythm* of a live source needs raw reading times, and only from the /// recent past: that query carries as its lower bound, so it plans over /// the newest raw chunks instead of every chunk a decade of history has. A live meter that delivered nothing /// inside the window has no rhythm there; those few meters are asked again over their whole history, so the /// stale verdict of a long-silent source is unchanged. /// private async Task LoadFreshnessAsync(CancellationToken cancellationToken) { var ids = _dataLeaves.ToList(); if (ids.Count == 0) { return; } var sources = await _db.MeterSources.AsNoTracking() .Include(s => s.Endpoint) .Where(s => ids.Contains(s.MeterId) && s.IsEnabled) .ToListAsync(cancellationToken).ConfigureAwait(false); var liveIds = ids.Where(id => sources.Exists(s => s.MeterId == id && IsLive(s))).ToList(); var readings = await AnalysisQueries .RecentReadingsAsync(_connection, liveIds, Now - FreshnessRules.RecentWindow, cancellationToken).ConfigureAwait(false); var silent = liveIds.Where(id => (readings.GetValueOrDefault(id)?.Count ?? 0) < 2).ToList(); if (silent.Count > 0) { foreach (var (id, times) in await AnalysisQueries .RecentReadingsAsync(_connection, silent, since: null, cancellationToken).ConfigureAwait(false)) { readings[id] = times; } } var events = await AnalysisQueries.LastEventsAsync(_connection, ids, cancellationToken).ConfigureAwait(false); foreach (var id in ids) { var times = readings.GetValueOrDefault(id) ?? []; var live = sources.Where(s => s.MeterId == id && IsLive(s)).ToList(); TimeSpan? poll = null; foreach (var source in live.Where(s => s.SourceType == SourceType.HomeAssistant)) { if (source.Endpoint is { } endpoint && HaEndpointConfig.Parse(endpoint.Config).UseWebSocket) { continue; } var interval = TimeSpan.FromMinutes(Math.Max(1, SourceConfig.Parse(source.Config).PollMinutes ?? 60)); poll = poll is { } p && p >= interval ? p : interval; } var input = new FreshnessInput( LastReadingOf(id, times), events.TryGetValue(id, out var lastEvent) ? lastEvent : null, times, live.Count > 0, poll); _freshness[id] = FreshnessRules.Evaluate(input, Now); } } /// A source that is expected to deliver on its own (D-18). private static bool IsLive(MeterSource source) => source.SourceType is SourceType.Mqtt or SourceType.Tasmota or SourceType.HomeAssistant; /// /// The meter's last reading time (A-40): what its stored analysis data recorded, and — for a live meter whose /// rhythm was read — whatever of the two is later, so a reading ingested since the last recompute still counts. /// private DateTimeOffset? LastReadingOf(int id, IReadOnlyList times) { var stored = _catalog.Meters.TryGetValue(id, out var meter) ? meter.State?.LastReadingAt : null; var sampled = times.Count > 0 ? times.Max() : (DateTimeOffset?)null; return stored is { } s && sampled is { } r ? (s >= r ? s : r) : stored ?? sampled; } // ---------------------------------------------------------------- physical values /// A physical meter's sums per bucket of one side, and their total. private (Tally[] Buckets, Tally Total) Tallies(Side side, LeafData leaf) { if (side.Tallies.TryGetValue(leaf.Id, out var cached)) { return cached; } var tallies = side.Buckets.Select(b => leaf.Sum(b.From, b.To, side.Cutoff)).ToArray(); var total = new Tally(); foreach (var tally in tallies) { total.Add(tally); } side.Tallies[leaf.Id] = (tallies, total); return (tallies, total); } /// /// A physical meter's values on one side: per bucket the sum with the status its coverage gives it (D-14), and the /// period total with the period's own status. As a contributor () the time outside its /// service period is a known zero (D-24). /// private PhysicalValues Physical(Side side, LeafData leaf, bool service) { var key = (leaf.Id, service); if (side.Physical.TryGetValue(key, out var cached)) { return cached; } var (tallies, total) = Tallies(side, leaf); PhysicalValues result; if (leaf.Meter.IsPending) { result = new PhysicalValues([.. side.Buckets.Select(_ => PendingValue)], PendingValue, null, null); } else { var runs = service ? leaf.ServiceRuns : leaf.Runs; var implied = leaf.Meter.Quantity.ImpliedProvenance; var coverage = CoverageEvaluator.EvaluateSeries(side.Buckets, runs, _zone, [.. tallies.Select(t => t.OpeningBalance)], side.Cutoff); var values = coverage.Select((c, i) => Withheld(c.ToValue(tallies[i].Amount, tallies[i].ProvenanceWith(implied)), tallies[i])).ToList(); var totalCoverage = CoverageEvaluator.Evaluate(side.PeriodBucket, runs, _zone, total.OpeningBalance, side.Cutoff); var totalValue = side == _current && _period.HasNotStarted() ? BucketValue.Missing(ValueIssue.NotYetOccurred) : Withheld(totalCoverage.ToValue(total.Amount, total.ProvenanceWith(implied)), total); result = new PhysicalValues(values, totalValue, coverage, totalCoverage); } side.Physical[key] = result; return result; } /// /// A bucket whose sum left out rows closing after the cutoff (A-05) is not complete, even where coverage reaches its /// end: a whole rollup day or month is withheld once one of its rows closes after the cutoff — a current-month label /// beside live readings takes the day's live share with it — so an available status would pass what is shown off /// as the whole bucket, or an empty day as a true zero (D-14, A-20). /// private static BucketValue Withheld(BucketValue value, Tally tally) => value.Status == BucketStatus.Available && tally.AfterRows > 0 ? value with { Status = BucketStatus.Partial, Issue = ValueIssue.RecordedAfterNow } : value; // ---------------------------------------------------------------- virtual values /// A virtual meter evaluated on one side (D-27); null when its definition cannot be evaluated. private VirtualEvaluation? Evaluate(Side side, int meterId) { if (side.Evaluations.TryGetValue(meterId, out var cached)) { return cached; } VirtualEvaluation? evaluation = null; if (_catalog.Find(meterId) is { IsVirtual: true, Formula: { } formula } meter) { var sources = formula.MeterIds.Select(id => SourceFor(side, id)).OfType().ToList(); var kind = meter.Validation?.Kind ?? meter.Quantity.Kind; evaluation = VirtualEvaluator.Evaluate(meterId, formula, kind, side.Buckets, sources); } side.Evaluations[meterId] = evaluation; return evaluation; } /// One source of a formula as the evaluator reads it, with the lifecycle of the source (D-24). private VirtualSource? SourceFor(Side side, int sourceId) { if (_catalog.Find(sourceId) is not { } source) { return null; } VirtualSource input; if (source.IsVirtual) { input = Evaluate(side, sourceId) is { } nested ? VirtualSource.FromEvaluation(nested) : _catalog.Graph.CycleFor(sourceId) is { } cycle ? VirtualSource.Failed(sourceId, BucketStatus.Invalid, ValueIssue.DependencyCycle, cycle) : VirtualSource.Failed(sourceId, BucketStatus.Invalid, ValueIssue.InvalidDefinition, [sourceId]); } else { input = PhysicalSource(side, _leaves[sourceId]); } return input with { InstalledAt = source.Meter.InstalledAt, RetiredAt = source.Meter.RetiredAt }; } /// /// A physical source for the evaluator (A-12): its figures per local day — covered or not from the coverage /// evaluator, the resolution and month division of the run covering the day, the day's actual amount — and its /// own bucket and period states, as a contributor. /// private VirtualSource PhysicalSource(Side side, LeafData leaf) { if (side.Sources.TryGetValue(leaf.Id, out var cached)) { return cached; } VirtualSource source; if (leaf.Meter.IsPending) { source = VirtualSource.Failed(leaf.Id, BucketStatus.Pending, ValueIssue.AnalysisPending, [leaf.Id]); } else { var windows = side.DayWindows(_zone); var dayBuckets = windows.Select(w => new AnalysisBucket(w.Day, w.Day.AddDays(1), w.From, w.To, BucketSize.Day)).ToList(); var coverage = CoverageEvaluator.EvaluateSeries(dayBuckets, leaf.Runs, _zone, null, side.Cutoff); var capped = leaf.CappedRuns(side.Cutoff).Where(r => !r.IsGap).OrderBy(r => r.From).ToList(); var implied = leaf.Meter.Quantity.ImpliedProvenance; var days = new Dictionary(); var cursor = 0; for (var i = 0; i < windows.Count; i++) { if (coverage[i].Covered <= TimeSpan.Zero) { continue; } var window = windows[i]; while (cursor < capped.Count && capped[cursor].To <= window.From) { cursor++; } var divided = true; var any = false; for (var r = cursor; r < capped.Count && capped[r].From < window.To; r++) { if (capped[r].To > window.From) { any = true; divided &= capped[r].DividedAtMonths; } } var tally = leaf.Sum(window.From, window.To, side.Cutoff); days[window.Day] = new SourceDay( tally.Amount, true, coverage[i].Resolution ?? ResolutionClass.Hour, tally.ProvenanceWith(implied), any && divided); } var states = Physical(side, leaf, service: true); source = new VirtualSource(leaf.Id, days) { BucketStates = states.Values, PeriodState = states.Total }; } side.Sources[leaf.Id] = source; return source; } // ---------------------------------------------------------------- series private AnalysisSeries MeterSeries(int meterId) { var meter = _catalog.Meters[meterId]; var key = SeriesKey.ForMeter(meterId, meter.EnergyTypeId, meter.Quantity.Unit); var entry = _catalog.Totals.Meters.GetValueOrDefault(meterId); if (!meter.IsVirtual) { var leaf = _leaves[meterId]; var asContributor = _request.AsContributors; var values = Physical(_current, leaf, service: asContributor); return new AnalysisSeries(key, meter.Name, SeriesBasis.Physical, meter.Quantity.Kind, meter.Quantity.Unit, values.Values, values.Total, true) { Coverage = values.Coverage, Resolution = values.TotalCoverage?.Resolution, Availability = AvailableRange.OfRuns(leaf.Runs, Now, _zone), Freshness = _freshness.GetValueOrDefault(meterId) ?? Freshness.None, RecordedAfterNow = AfterNowFor([meterId]), Totals = entry, Notes = meter.Quantity.Notes, Comparison = Compare(MeterKey(meterId), side => { var v = Physical(side, leaf, service: asContributor); return (v.Values, v.Total); }, side => MatchedValue(side, meterId)), }; } var info = VirtualInfo(meter); var leaves = LeavesOf(meterId); var basis = meter.VirtualStatus == VirtualMeterStatus.Legacy ? SeriesBasis.LegacyVirtual : SeriesBasis.Virtual; var evaluation = Evaluate(_current, meterId); var series = evaluation is null ? new AnalysisSeries(key, meter.Name, basis, meter.Quantity.Kind, meter.Quantity.Unit, Invalid(meter, _current.Buckets.Count), InvalidValue(meter), false) : new AnalysisSeries( key, meter.Name, basis, meter.Quantity.Kind, meter.Quantity.Unit, MarkLegacy(meter, evaluation.Values), MarkLegacy(meter, evaluation.Total), evaluation.IsAdditive) { Resolution = evaluation.Resolution, Contributions = Contributions(_current, evaluation, [meterId]), }; return series with { Availability = VirtualAvailability(meterId), Freshness = FreshnessRules.Combine(leaves.Select(l => _freshness.GetValueOrDefault(l) ?? Freshness.None)), RecordedAfterNow = AfterNowFor(leaves), Totals = entry, Virtual = info, Notes = meter.Quantity.Notes, Comparison = Compare(MeterKey(meterId), side => VirtualValues(side, meter), side => MatchedValue(side, meterId)), }; } private AnalysisSeries MeasureSeries(int energyTypeId, MeasureGroup group) { var key = SeriesKey.ForMeasure(energyTypeId, group.Measure, group.Unit); var (values, total) = MeasureValuesOf(_current, group.MeterIds); var leaves = group.MeterIds.SelectMany(LeavesOf).Distinct().ToList(); return new AnalysisSeries(key, string.Empty, SeriesBasis.Measure, KindOf(group.Measure), group.Unit, values, total, true) { MemberIds = group.MeterIds, Resolution = MeasureResolution(group.MeterIds), Availability = AvailableRange.Union(group.MeterIds.Select(MeterAvailability), _zone), Freshness = FreshnessRules.Combine(leaves.Select(l => _freshness.GetValueOrDefault(l) ?? Freshness.None)), RecordedAfterNow = AfterNowFor(leaves), Comparison = Compare( MeasureKey(energyTypeId, group), side => MeasureValuesOf(side, group.MeterIds), side => group.MeterIds.Select(id => MatchedValue(side, id)).Aggregate((double?)0d, (sum, v) => sum is { } s && v is { } x ? s + x : null)), }; } /// /// The coarsest resolution among a measure's members (D-51): each as its own series reports it — a physical meter's /// coverage over the period, a virtual meter's evaluation (divided runs are at most monthly, A-03). A total can be /// opened no finer than its coarsest member resolves. /// private ResolutionClass? MeasureResolution(IReadOnlyList members) { ResolutionClass? coarsest = null; foreach (var id in members) { var resolution = _catalog.Meters[id].IsVirtual ? Evaluate(_current, id)?.Resolution : Physical(_current, _leaves[id], service: true).TotalCoverage?.Resolution; if (resolution is { } value && (coarsest is null || value > coarsest)) { coarsest = value; } } return coarsest; } /// A measure's values on one side: its members' values as contributors, added up (D-22). private (IReadOnlyList Values, BucketValue Total) MeasureValuesOf(Side side, IReadOnlyList members) { var parts = members.Select(id => (id, Values: MemberValues(side, id))).ToList(); var values = MeasureValues.SumSeries([.. parts.Select(p => (p.id, p.Values.Values))], side.Buckets.Count); var total = MeasureValues.Sum([.. parts.Select(p => (p.id, p.Values.Total))]); if (side == _current && _period.HasNotStarted()) { total = BucketValue.Missing(ValueIssue.NotYetOccurred); } return (values, total); } private (IReadOnlyList Values, BucketValue Total) MemberValues(Side side, int meterId) { var meter = _catalog.Meters[meterId]; if (!meter.IsVirtual) { var values = Physical(side, _leaves[meterId], service: true); return (values.Values, values.Total); } return VirtualValues(side, meter); } private (IReadOnlyList Values, BucketValue Total) VirtualValues(Side side, AnalysisMeter meter) { var evaluation = Evaluate(side, meter.Id); return evaluation is null ? (Invalid(meter, side.Buckets.Count), InvalidValue(meter)) : (MarkLegacy(meter, evaluation.Values), MarkLegacy(meter, evaluation.Total)); } /// The series on the comparison side, and the change over the coverage both share (D-07, D-08). private SeriesComparison? Compare( string key, Func Values, BucketValue Total)> valuesOn, Func matchedOn) { if (_comparison is null) { return null; } var (values, total) = valuesOn(_comparison); var matched = _matched.GetValueOrDefault(key) ?? MatchedCoverageResult.NotComparable; if (!matched.IsComparable) { return new SeriesComparison(values, total, matched, null, null, Change.Unavailable); } var current = matchedOn(new Side(MatchedWindows(matched, comparison: false), _current.PeriodBucket, _current.Cutoff)); var previous = matchedOn(new Side(MatchedWindows(matched, comparison: true), _comparison.PeriodBucket, _comparison.Cutoff)); return new SeriesComparison(values, total, matched, current, previous, Change.Between(current, previous)); } /// The matched pieces of one side as windows to sum. private static List MatchedWindows(MatchedCoverageResult matched, bool comparison) => [.. matched.Pieces.Select(p => comparison ? p.Comparison : p.Current).Select(r => new AnalysisBucket(r.FirstDay, r.LastDay.AddDays(1), r.From, r.To, BucketSize.Day))]; /// /// A meter's value over a side's windows (here: matched pieces): a physical meter's actual sum, a virtual meter's /// formula applied to its sources' sums — the joint coverage the matched range is (D-27). Null when not evaluable. /// private double? MatchedValue(Side windows, int meterId) { if (_catalog.Find(meterId) is not { } meter) { return null; } if (!meter.IsVirtual) { var leaf = _leaves[meterId]; if (leaf.Meter.IsPending) { return null; } return windows.Buckets.Sum(w => leaf.Sum(w.From, w.To, windows.Cutoff).Amount); } if (meter.Formula is not { } formula) { return null; } var value = formula.Evaluate(id => MatchedValue(windows, id) ?? double.NaN); return double.IsFinite(value) ? value : null; } private List Contributions(Side side, VirtualEvaluation evaluation, IReadOnlyList path) => [.. evaluation.Contributions.Select(c => { var source = _catalog.Find(c.MeterId); var isVirtual = source?.IsVirtual == true; IReadOnlyList sourcePath = [.. path, c.MeterId]; var nested = isVirtual && !path.Contains(c.MeterId) && Evaluate(side, c.MeterId) is { } inner ? Contributions(side, inner, sourcePath) : []; return new SeriesContribution( c.MeterId, source?.Name ?? string.Empty, isVirtual, c.Coefficient, c.Values, c.UsedAmounts, c.Total, c.UsedTotal, sourcePath, nested); })]; private VirtualSeriesInfo VirtualInfo(AnalysisMeter meter) { var definition = meter.Definition; return new VirtualSeriesInfo( meter.VirtualStatus ?? VirtualMeterStatus.NeedsConfiguration, definition?.Expression, meter.CostRule, meter.Formula?.MeterIds ?? definition?.ReferencedMeterIds ?? [], LeavesOf(meter.Id), meter.Validation?.Problems ?? [], meter.Legacy, meter.VirtualStatus == VirtualMeterStatus.Malformed ? meter.StoredDefinition?.Problem : null); } /// A virtual meter's own buckets when it cannot be evaluated: invalid, naming why (D-26, D-28). private IReadOnlyList Invalid(AnalysisMeter meter, int count) { var value = InvalidValue(meter); return [.. Enumerable.Repeat(value, count)]; } private BucketValue InvalidValue(AnalysisMeter meter) { var id = meter.Id; if (_catalog.Graph.CycleFor(id) is { } cycle) { return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, cycle); } var problem = meter.Validation?.Problems.FirstOrDefault(); if (problem is { Kind: VirtualProblemKind.DependencyCycle }) { IReadOnlyList path = problem.MeterIds.Count > 0 && problem.MeterIds[0] == id ? problem.MeterIds : [id, .. problem.MeterIds]; return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, path); } if (meter.Legacy is { Outcome: LegacyDerivationOutcome.Cycle } legacy) { IReadOnlyList path = legacy.MeterIds.Count > 0 && legacy.MeterIds[0] == id ? legacy.MeterIds : [id, .. legacy.MeterIds]; return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, path); } IReadOnlyList cause = problem is { MeterIds.Count: > 0 } && problem.MeterIds[0] != id ? [id, problem.MeterIds[0]] : [id]; return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.InvalidDefinition, null, cause); } /// A legacy meter's evaluated values carry "legacy — confirm" where nothing more important is said (D-28). private static IReadOnlyList MarkLegacy(AnalysisMeter meter, IReadOnlyList values) => meter.VirtualStatus == VirtualMeterStatus.Legacy ? [.. values.Select(v => MarkLegacy(meter, v))] : values; private static BucketValue MarkLegacy(AnalysisMeter meter, BucketValue value) => meter.VirtualStatus == VirtualMeterStatus.Legacy && value.Issue == ValueIssue.None ? value with { Issue = ValueIssue.LegacyDefinition } : value; private static QuantityKind KindOf(TotalsMeasure measure) => measure switch { TotalsMeasure.Generation => QuantityKind.Generation, TotalsMeasure.Export => QuantityKind.Export, TotalsMeasure.Runtime => QuantityKind.Runtime, _ => QuantityKind.Consumption, }; // ---------------------------------------------------------------- recorded after now, availability, problems /// /// A physical meter's rows in the requested range that close after now (D-04, A-05): rollups left out of the /// current sums, today's rows after now, and the days after today up to the end of the named range. /// private RecordedAfterNow? AfterNowOf(LeafData leaf) { var (_, total) = Tallies(_current, leaf); var block = new Tally(); block.Add(total); var rows = block.AfterRows; var amount = block.AfterAmount; var first = block.AfterFirstDay; var last = block.AfterLastDay; void Add(int count, double sum, DateOnly day) { if (count <= 0) { return; } rows += count; amount += sum; first = first is { } f && f <= day ? f : day; last = last is { } l && l >= day ? l : day; } if (_readsToday) { var today = RangeParts.LocalDate(Now, _zone); var later = leaf.RawFrom(today, Now).ToList(); Add(later.Count, later.Sum(r => r.Amount), today); } foreach (var (day, rollup) in leaf.AfterNowDays) { Add(rollup.Rows, rollup.Amount, day); } return rows > 0 && first is { } firstDay && last is { } lastDay ? new RecordedAfterNow(leaf.Id, rows, amount, firstDay, lastDay) : null; } private IReadOnlyList AfterNowFor(IEnumerable leaves) => [.. leaves.Select(id => _afterNow.GetValueOrDefault(id)).OfType()]; /// A meter's data range (D-19): a physical meter's coverage, a virtual meter's joint coverage. private AvailableRange? MeterAvailability(int meterId) { if (_catalog.Find(meterId) is not { } meter) { return null; } return meter.IsVirtual ? VirtualAvailability(meterId) : _leaves.TryGetValue(meterId, out var leaf) ? AvailableRange.OfRuns(leaf.Runs, Now, _zone) : null; } /// /// Where every source of a virtual meter has data (D-27): the intersection of their coverage, with the time outside /// a source's service period counted as its known zero (D-24), within the outer bounds of the sources' own data. /// private AvailableRange? VirtualAvailability(int meterId) { var leaves = LeavesOf(meterId).Where(_leaves.ContainsKey).Select(id => _leaves[id]).ToList(); if (leaves.Count == 0) { return null; } IReadOnlyList> capped = [.. leaves.Select(l => CoverageRuns.CapAt(l.ServiceRuns, Now, _zone))]; var joint = CoverageRuns.Covered(CoverageRuns.Intersect(capped)); var hull = AvailableRange.Union(leaves.Select(l => AvailableRange.OfRuns(l.Runs, Now, _zone)), _zone); if (joint is not { First: { } from, Last: { } to } || hull is null) { return null; } return AvailableRange.Of(from > hull.From ? from : hull.From, to < hull.To ? to : hull.To, _zone); } /// The scope's availability (D-19): its meters' data, and for cost its billed meters' data plus manual costs. private async Task AvailabilityAsync(CancellationToken cancellationToken) { var quantity = AvailableRange.Union(ScopeMeters().Select(MeterAvailability), _zone); if (_request.QuantitiesOnly) { return new ScopeAvailability(quantity, null, null); } var metered = AvailableRange.Union(CostMeters().Select(MeterAvailability), _zone); var today = RangeParts.LocalDate(Now, _zone); var manual = await _db.ManualCosts.AsNoTracking() .Where(c => c.PeriodStart <= today) .Select(c => new { c.MeterId, c.PeriodStart }) .ToListAsync(cancellationToken).ConfigureAwait(false); var scopeMeters = ScopeMeters().ToHashSet(); var days = manual .Where(c => _request.Scope.Kind == AnalysisScopeKind.Portfolio || (c.MeterId is { } id && scopeMeters.Contains(id))) .Select(c => c.PeriodStart) .ToList(); AvailableRange? manualRange = null; if (days.Count > 0) { var from = Core.Normalization.GapAttribution.LocalMidnight(days.Min(), _zone); var to = Core.Normalization.GapAttribution.LocalMidnight(days.Max().AddDays(1), _zone); manualRange = AvailableRange.Of(from, to < Now ? to : Now, _zone) ?? AvailableRange.Of(from, to, _zone); } var cost = AvailableRange.Union([metered, manualRange], _zone); LatestPeriod? latest = null; if (cost is not null) { var month = cost.LatestMonth; var byMeters = metered?.LatestMonth == month; var byManual = manualRange?.LatestMonth == month; latest = new LatestPeriod(month, byMeters && byManual ? LatestPeriodBasis.Both : byManual ? LatestPeriodBasis.Manual : LatestPeriodBasis.Meters); } return new ScopeAvailability(quantity, cost, latest); } /// The attention items (D-53) of everything the result shows. private void CollectProblems() { var shown = _seriesIds.Concat(_measures.SelectMany(m => m.Group.MeterIds)).Distinct().ToList(); var virtuals = new SortedSet(); foreach (var id in shown) { CollectVirtuals(id, virtuals); } foreach (var id in virtuals) { var meter = _catalog.Meters[id]; switch (meter.VirtualStatus) { case VirtualMeterStatus.Legacy: _problems.Add(new AnalysisProblem(AnalysisProblemKind.LegacyDefinition, id) { MeterIds = meter.Legacy?.MeterIds ?? [], Legacy = meter.Legacy?.Outcome, }); break; case VirtualMeterStatus.NeedsConfiguration: _problems.Add(new AnalysisProblem(AnalysisProblemKind.LegacyNeedsConfiguration, id) { MeterIds = meter.Legacy?.MeterIds ?? [], Values = meter.Legacy?.Values ?? [], Legacy = meter.Legacy?.Outcome, }); break; case VirtualMeterStatus.Malformed: _problems.Add(new AnalysisProblem(AnalysisProblemKind.MalformedDefinition, id) { Values = meter.StoredDefinition?.Problem is { } malformation ? [malformation] : [], }); break; case VirtualMeterStatus.Invalid: foreach (var problem in meter.Validation?.Problems ?? []) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.InvalidDefinition, id) { MeterIds = problem.MeterIds, Values = problem.Values, Virtual = problem, }); } break; } } foreach (var id in _dataLeaves.Order()) { if (_leaves[id].Meter.IsPending) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.AnalysisPending, id)); } if (_afterNow.GetValueOrDefault(id) is { } block) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.RecordedAfterNow, id) { AfterNow = block }); } if (_freshness.GetValueOrDefault(id) is { State: FreshnessState.Stale }) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.StaleSource, id)); } } if (_request.Scope.Kind == AnalysisScopeKind.Meters) { return; } var types = TypesInScope(); foreach (var problem in _catalog.Totals.Problems.Where(p => _catalog.Find(p.MeterId) is { } m && types.Contains(m.EnergyTypeId))) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.TotalsProblem, problem.MeterId) { MeterIds = problem.OtherMeterId is { } other ? [other] : [], Totals = problem, }); } foreach (var hint in _catalog.Totals.Hints.Where(h => types.Contains(h.EnergyTypeId))) { _problems.Add(new AnalysisProblem(AnalysisProblemKind.PossibleOverlap, hint.MeterId) { MeterIds = [hint.OtherMeterId], Hint = hint }); } } private void CollectVirtuals(int meterId, SortedSet found) { if (_catalog.Find(meterId) is not { IsVirtual: true } meter || !found.Add(meterId)) { return; } foreach (var source in meter.Formula?.MeterIds ?? []) { CollectVirtuals(source, found); } } private HashSet TypesInScope() => _request.Scope.Kind == AnalysisScopeKind.EnergyType ? [_request.Scope.EnergyTypeId!.Value] : [.. _catalog.Meters.Values.Select(m => m.EnergyTypeId)]; private IReadOnlyList Classification() { if (_request.Scope.Kind == AnalysisScopeKind.Meters) { return []; } return [.. ScopeMeters().Order() .Where(id => _catalog.Totals.Meters.ContainsKey(id)) .Select(id => new MeterClassification(id, _catalog.Meters[id].Name, _catalog.Totals.Meters[id]))]; } private static List Deduplicated(List problems) { var seen = new HashSet<(AnalysisProblemKind, int?, VirtualProblemKind?, TotalsProblemKind?, OverlapHintKind?)>(); return [.. problems.Where(p => seen.Add((p.Kind, p.MeterId, p.Virtual?.Kind, p.Totals?.Kind, p.Hint?.Kind)))]; } private static string MeterKey(int meterId) => SeriesKey.ForMeter(meterId, 0, string.Empty).Id; private static string MeasureKey(int energyTypeId, MeasureGroup group) => SeriesKey.ForMeasure(energyTypeId, group.Measure, group.Unit).Id; /// The values of one physical meter on one side. private sealed record PhysicalValues( IReadOnlyList Values, BucketValue Total, IReadOnlyList? Coverage, BucketCoverage? TotalCoverage); /// /// One side of a request — the current period or its comparison — with its buckets, the bucket that is the whole /// period, the cut-off its actuals stop at, and what has been computed for it. /// private sealed class Side(IReadOnlyList buckets, AnalysisBucket periodBucket, DateTimeOffset cutoff) { private List? _dayWindows; public IReadOnlyList Buckets { get; } = buckets; public AnalysisBucket PeriodBucket { get; } = periodBucket; public DateTimeOffset Cutoff { get; } = cutoff; public Dictionary Tallies { get; } = []; public Dictionary<(int, bool), PhysicalValues> Physical { get; } = []; public Dictionary Evaluations { get; } = []; public Dictionary Sources { get; } = []; /// The local days the buckets span, each clipped to the buckets' range. public List DayWindows(TimeZoneInfo zone) { if (_dayWindows is not null) { return _dayWindows; } _dayWindows = []; if (Buckets.Count == 0) { return _dayWindows; } var from = Buckets[0].From; var to = Buckets[^1].To; if (to <= from) { return _dayWindows; } for (var day = RangeParts.LocalDate(from, zone); ; day = day.AddDays(1)) { var start = Core.Normalization.GapAttribution.LocalMidnight(day, zone); if (start >= to) { break; } var end = Core.Normalization.GapAttribution.LocalMidnight(day.AddDays(1), zone); var windowFrom = start > from ? start : from; var windowTo = end < to ? end : to; if (windowTo > windowFrom) { _dayWindows.Add(new DayWindow(day, windowFrom, windowTo)); } } return _dayWindows; } } /// One local day of a side, clipped to its range. private readonly record struct DayWindow(DateOnly Day, DateTimeOffset From, DateTimeOffset To); }