using System.Data.Common;
using MeterVault.Core.Analysis;
using MeterVault.Core.Analysis.Coverage;
using MeterVault.Core.Analysis.Totals;
using MeterVault.Core.Analysis.Virtual;
using MeterVault.Core.Domain;
using MeterVault.Infrastructure.Ingestion;
using MeterVault.Infrastructure.Persistence;
using Microsoft.EntityFrameworkCore;
namespace MeterVault.Infrastructure.Analysis;
///
/// One analysis request from catalog to result: which series and meters it needs, what to read for them, and how the
/// read data becomes bucket values (D-14, D-27), totals (D-22), comparisons (D-07) and availability (D-19).
///
///
/// The order matters and is fixed: targets → physical leaves → coverage → bucket plan and comparison → matched
/// coverage → every window a sum will need is registered → the rollups and edge rows are loaded, once per table →
/// values are computed from memory. Registering before loading is what keeps the reader to one query per table while
/// still summing arbitrary ranges (a matched range can end mid-day).
///
internal sealed class AnalysisRun
{
private static readonly BucketValue PendingValue = new(null, BucketStatus.Pending, Provenance.None, ValueIssue.AnalysisPending);
private readonly MeterVaultDbContext _db;
private readonly DbConnection _connection;
private readonly AnalysisCatalog _catalog;
private readonly AnalysisRequest _request;
private readonly ResolvedPeriod _period;
private readonly TimeZoneInfo _zone;
/// Every physical meter whose coverage is read (availability needs the whole scope).
private readonly Dictionary _leaves = [];
/// The physical meters whose rollups are read (series, members and virtual sources).
private readonly HashSet _dataLeaves = [];
private readonly Dictionary> _leavesOf = [];
private readonly Dictionary _freshness = [];
private readonly Dictionary _afterNow = [];
private readonly List _problems = [];
private List _seriesIds = [];
private List<(int EnergyTypeId, MeasureGroup Group)> _measures = [];
private Side _current = null!;
private Side? _comparison;
private ComparisonResolution? _resolution;
private ResolvedPeriod? _comparisonPeriod;
private IReadOnlyList _pairs = [];
private readonly Dictionary _matched = [];
private bool _readsToday;
public AnalysisRun(MeterVaultDbContext db, AnalysisCatalog catalog, AnalysisRequest request, TimeZoneInfo zone)
{
_db = db;
_connection = db.Database.GetDbConnection();
_catalog = catalog;
_request = request;
_period = request.Period;
_zone = zone;
}
private DateTimeOffset Now => _period.Now;
/// The physical meters this run read, with what it registered and loaded for each (for tests of the D-15 budget).
internal IReadOnlyDictionary Leaves => _leaves;
public async Task ExecuteAsync(BucketPlan? plan, CancellationToken cancellationToken)
{
ChooseTargets();
CollectLeaves(includeScope: true);
await LoadCoverageAsync(cancellationToken).ConfigureAwait(false);
plan ??= BucketPlanner.Plan(_period, BucketSize.Auto, CoarsestNeeded(), _request.MaxPoints);
if (plan.Refused)
{
return new AnalysisResult(_request, plan, [], [], ScopeAvailability.None, []) { Refusal = AnalysisRefusal.TooManyPoints };
}
foreach (var id in _dataLeaves)
{
_leaves[id].UseMonths = plan.Size is BucketSize.Month or BucketSize.Year && !_leaves[id].IsVirtualSource;
}
_current = new Side(plan.Buckets, PeriodBucket.Of(_period), Now);
await ResolveComparisonAsync(plan, cancellationToken).ConfigureAwait(false);
MatchCoverage();
RegisterWindows();
await LoadDataAsync(cancellationToken).ConfigureAwait(false);
if (!_request.QuantitiesOnly)
{
await LoadFreshnessAsync(cancellationToken).ConfigureAwait(false);
}
foreach (var id in _dataLeaves)
{
if (AfterNowOf(_leaves[id]) is { } block)
{
_afterNow[id] = block;
}
}
var series = _seriesIds.Select(MeterSeries).ToList();
var measures = _measures.Select(m => MeasureSeries(m.EnergyTypeId, m.Group)).ToList();
var availability = await AvailabilityAsync(cancellationToken).ConfigureAwait(false);
CollectProblems();
return new AnalysisResult(_request, plan, series, measures, availability, Deduplicated(_problems))
{
Comparison = _resolution is null ? null : new AnalysisComparison(_resolution, _comparisonPeriod, _pairs),
Classification = Classification(),
};
}
///
/// The buckets chooses for the request (D-05) from the coverage of the data it reads —
/// the plan would make, without reading any rollups.
///
public async Task PlanAsync(CancellationToken cancellationToken)
{
ChooseTargets();
CollectLeaves(includeScope: false);
await LoadCoverageAsync(cancellationToken).ConfigureAwait(false);
return BucketPlanner.Plan(_period, BucketSize.Auto, CoarsestNeeded(), _request.MaxPoints);
}
public async Task AvailabilityOnlyAsync(CancellationToken cancellationToken)
{
CollectLeaves(includeScope: true);
await LoadCoverageAsync(cancellationToken).ConfigureAwait(false);
return await AvailabilityAsync(cancellationToken).ConfigureAwait(false);
}
// ---------------------------------------------------------------- targets and leaves
/// The meter series and measure totals the scope asks for.
private void ChooseTargets()
{
var scope = _request.Scope;
switch (scope.Kind)
{
case AnalysisScopeKind.Meters:
foreach (var id in scope.MeterIds)
{
if (_catalog.Find(id) is null)
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.UnknownMeter, id));
}
else
{
_seriesIds.Add(id);
}
}
break;
case AnalysisScopeKind.EnergyType:
var type = scope.EnergyTypeId!.Value;
_measures = [.. _catalog.Totals.ForType(type).Measures.Select(g => (type, g))];
if (_request.IncludeMeterSeries)
{
_seriesIds = [.. ScopeMeters().OrderBy(id => id)];
}
break;
default:
_measures = [.. _catalog.Totals.Types.OrderBy(t => t.Key).SelectMany(t => t.Value.Measures.Select(g => (t.Key, g)))];
if (_request.IncludeMeterSeries)
{
_seriesIds = [.. ScopeMeters().OrderBy(id => id)];
}
break;
}
}
/// The meters of the scope: the selection, the energy type's meters, or every meter.
private IEnumerable ScopeMeters() => _request.Scope.Kind switch
{
AnalysisScopeKind.Meters => _request.Scope.MeterIds.Where(id => _catalog.Find(id) is not null),
AnalysisScopeKind.EnergyType => _catalog.Meters.Values.Where(m => m.EnergyTypeId == _request.Scope.EnergyTypeId).Select(m => m.Id),
_ => _catalog.Meters.Keys,
};
/// The meters whose cost the scope is about (D-19): the selection, or the billed meters of the type(s).
private IEnumerable CostMeters() => _request.Scope.Kind switch
{
AnalysisScopeKind.Meters => ScopeMeters(),
AnalysisScopeKind.EnergyType => _catalog.Totals.ForType(_request.Scope.EnergyTypeId!.Value).Billing.Items,
_ => _catalog.Totals.BillItems,
};
///
/// Expands the targets to the physical meters they read, through evaluable virtual definitions only. Data leaves
/// feed values; the scope's and the bill's other meters only need their coverage, for availability.
///
private void CollectLeaves(bool includeScope)
{
var dataTargets = _seriesIds.Concat(_measures.SelectMany(m => m.Group.MeterIds)).Distinct().ToList();
foreach (var id in dataTargets)
{
foreach (var leaf in LeavesOf(id))
{
_dataLeaves.Add(leaf);
Leaf(leaf);
}
MarkVirtualSources(id, []);
}
if (!includeScope)
{
return;
}
foreach (var id in ScopeMeters().Concat(CostMeters()).Distinct())
{
foreach (var leaf in LeavesOf(id))
{
Leaf(leaf);
}
}
}
/// The physical meters a meter reads: itself, or the leaves of its evaluable formula; none when not evaluable.
private IReadOnlyList LeavesOf(int meterId)
{
if (_leavesOf.TryGetValue(meterId, out var cached))
{
return cached;
}
var leaves = new SortedSet();
var seen = new HashSet();
var pending = new Stack([meterId]);
while (pending.Count > 0)
{
var id = pending.Pop();
if (!seen.Add(id) || _catalog.Find(id) is not { } meter)
{
continue;
}
if (!meter.IsVirtual)
{
leaves.Add(id);
continue;
}
foreach (var source in meter.Formula?.MeterIds ?? [])
{
pending.Push(source);
}
}
IReadOnlyList result = [.. leaves];
_leavesOf[meterId] = result;
return result;
}
/// Marks the physical meters a virtual meter reads directly or through nested ones: they are read day by day.
private void MarkVirtualSources(int meterId, HashSet seen)
{
if (!seen.Add(meterId) || _catalog.Find(meterId) is not { IsVirtual: true, Formula: { } formula })
{
return;
}
foreach (var source in formula.MeterIds)
{
if (_catalog.Find(source) is { IsVirtual: false })
{
Leaf(source).IsVirtualSource = true;
}
else
{
MarkVirtualSources(source, seen);
}
}
}
private LeafData Leaf(int meterId)
{
if (!_leaves.TryGetValue(meterId, out var leaf))
{
_leaves[meterId] = leaf = new LeafData(_catalog.Meters[meterId], _zone);
}
return leaf;
}
private async Task LoadCoverageAsync(CancellationToken cancellationToken)
{
var runs = await AnalysisQueries.CoverageAsync(_connection, _leaves.Keys, cancellationToken).ConfigureAwait(false);
foreach (var (id, leaf) in _leaves)
{
leaf.Runs = runs.GetValueOrDefault(id) ?? [];
}
}
///
/// The coarsest resolution among the data covering the period (D-05): auto never plans finer. What each run asks
/// for is — a run divided at month boundaries resolves
/// months whatever its class (A-03), and a run coarser than a month only pushes the chart to years when years
/// can place it (A-41).
///
private ResolutionClass? CoarsestNeeded() =>
ResolutionClassifier.PlanningResolution(
_dataLeaves
.SelectMany(id => _leaves[id].CappedRuns(Now))
.Where(run => !run.IsGap && run.From < _period.To && run.To > _period.From),
_zone);
// ---------------------------------------------------------------- comparison and matched coverage
private async Task ResolveComparisonAsync(BucketPlan plan, CancellationToken cancellationToken)
{
if (_request.Comparison.Kind == ComparisonKind.None)
{
return;
}
_resolution = ComparisonResolver.Resolve(_period, _request.Comparison);
if (!_resolution.IsApplicable)
{
return;
}
var comparison = _resolution.Period;
_comparisonPeriod = comparison.ToResolvedPeriod(_period);
_pairs = ComparisonResolver.PairBuckets(_period, comparison, plan.Buckets);
_comparison = new Side([.. _pairs.Select(p => p.Comparison)], PeriodBucket.Of(_comparisonPeriod), _comparisonPeriod.Now);
var stamps = await AnalysisQueries.OpeningBalancesAsync(_connection, _dataLeaves, cancellationToken).ConfigureAwait(false);
foreach (var (id, stamp) in stamps)
{
_leaves[id].OpeningBalanceStamp = stamp;
}
}
/// The coverage both periods share (D-07), per series: the meter's own, or all its sources jointly.
private void MatchCoverage()
{
if (_comparison is null || _resolution?.Period is not { } comparison)
{
return;
}
foreach (var id in _seriesIds)
{
var meter = _catalog.Meters[id];
var leaves = LeavesOf(id).Select(l => _leaves[l]).ToList();
var own = !meter.IsVirtual && !_request.AsContributors;
_matched[MeterKey(id)] = Match(comparison, leaves, own);
}
foreach (var (type, group) in _measures)
{
var leaves = group.MeterIds.SelectMany(LeavesOf).Distinct().Select(l => _leaves[l]).ToList();
_matched[MeasureKey(type, group)] = Match(comparison, leaves, own: false);
}
}
private MatchedCoverageResult Match(ComparisonPeriod comparison, List leaves, bool own)
{
if (leaves.Count == 0 || leaves.Exists(l => l.Meter.IsPending))
{
return MatchedCoverageResult.NotComparable;
}
IReadOnlyList> sources = [.. leaves.Select(l => own ? l.Runs : l.ServiceRuns)];
IReadOnlyList balances = [.. leaves.Where(l => l.OpeningBalanceStamp is not null).Select(l => l.OpeningBalanceStamp!.Value)];
var current = MatchSide.OfSources(_period.From, _period.To, Now, _zone, sources, balances);
var previous = MatchSide.OfSources(comparison.From, comparison.To, _comparison!.Cutoff, _zone, sources, balances);
return MatchedCoverage.Match(current, previous, instant => comparison.MapInstant(instant, _zone));
}
// ---------------------------------------------------------------- registering and loading
/// Registers every window a sum will read, so the loads fetch exactly those rows.
private void RegisterWindows()
{
List sides = [_current];
if (_comparison is not null)
{
sides.Add(_comparison);
}
foreach (var id in _dataLeaves)
{
var leaf = _leaves[id];
foreach (var side in sides)
{
foreach (var bucket in side.Buckets)
{
leaf.Request(bucket.From, bucket.To);
}
if (leaf.IsVirtualSource)
{
foreach (var window in side.DayWindows(_zone))
{
leaf.Request(window.From, window.To);
}
}
}
}
// Matched pieces are only summed: their partial days are summed in the database, not loaded (D-15).
foreach (var (key, matched) in _matched)
{
foreach (var leaf in LeavesOfKey(key))
{
foreach (var piece in matched.Pieces)
{
leaf.RequestSummed(piece.Current.From, piece.Current.To, _current.Cutoff);
leaf.RequestSummed(piece.Comparison.From, piece.Comparison.To, _comparison!.Cutoff);
}
}
}
// Rows of today that close after now go to the "recorded after now" block (D-04).
var today = RangeParts.LocalDate(Now, _zone);
_readsToday = ReachesNow() && _period.FirstDay <= today && (_period.Preset == PeriodPreset.AllHistory || _period.NominalLastDay() >= today);
if (_readsToday)
{
foreach (var id in _dataLeaves)
{
_leaves[id].RequestRawDay(today);
}
}
}
private IEnumerable LeavesOfKey(string key)
{
var ids = key.StartsWith('m')
? LeavesOf(_seriesIds.First(id => MeterKey(id) == key))
: _measures.Where(m => MeasureKey(m.EnergyTypeId, m.Group) == key).SelectMany(m => m.Group.MeterIds).SelectMany(LeavesOf).Distinct();
return ids.Select(id => _leaves[id]);
}
/// True when the requested range reaches now or lies after it: then rows may close after now (D-04).
private bool ReachesNow() => _period.IsToDate || _period.ExtendsPastNow || _period.HasNotStarted();
private async Task LoadDataAsync(CancellationToken cancellationToken)
{
var data = _dataLeaves.ToDictionary(id => id, id => _leaves[id]);
var today = RangeParts.LocalDate(Now, _zone);
var afterFrom = _period.FirstDay > today ? _period.FirstDay : today.AddDays(1);
DateOnly? afterTo = _period.Preset == PeriodPreset.AllHistory ? null : _period.NominalLastDay().AddDays(1);
IReadOnlyCollection afterIds = ReachesNow() ? _dataLeaves : [];
await AnalysisQueries.DayRollupsAsync(_connection, data, afterIds, afterFrom, afterTo, Now, cancellationToken).ConfigureAwait(false);
await AnalysisQueries.MonthRollupsAsync(_connection, data, cancellationToken).ConfigureAwait(false);
await AnalysisQueries.RawDaysAsync(_connection, data, _zone, cancellationToken).ConfigureAwait(false);
await AnalysisQueries.WindowSumsAsync(_connection, data, cancellationToken).ConfigureAwait(false);
foreach (var leaf in data.Values)
{
leaf.Seal();
}
}
/// Freshness of every data leaf (D-18): the last reading and event, and the live sources' rhythm.
///
/// The mark — the meter's last reading — is read from its stored rollup state, which the recompute behind every
/// write path records (A-40); an import-only meter's mark is years old and must stay exact, so it is never
/// guessed from a bounded sample. Only the *rhythm* of a live source needs raw reading times, and only from the
/// recent past: that query carries as its lower bound, so it plans over
/// the newest raw chunks instead of every chunk a decade of history has. A live meter that delivered nothing
/// inside the window has no rhythm there; those few meters are asked again over their whole history, so the
/// stale verdict of a long-silent source is unchanged.
///
private async Task LoadFreshnessAsync(CancellationToken cancellationToken)
{
var ids = _dataLeaves.ToList();
if (ids.Count == 0)
{
return;
}
var sources = await _db.MeterSources.AsNoTracking()
.Include(s => s.Endpoint)
.Where(s => ids.Contains(s.MeterId) && s.IsEnabled)
.ToListAsync(cancellationToken).ConfigureAwait(false);
var liveIds = ids.Where(id => sources.Exists(s => s.MeterId == id && IsLive(s))).ToList();
var readings = await AnalysisQueries
.RecentReadingsAsync(_connection, liveIds, Now - FreshnessRules.RecentWindow, cancellationToken).ConfigureAwait(false);
var silent = liveIds.Where(id => (readings.GetValueOrDefault(id)?.Count ?? 0) < 2).ToList();
if (silent.Count > 0)
{
foreach (var (id, times) in await AnalysisQueries
.RecentReadingsAsync(_connection, silent, since: null, cancellationToken).ConfigureAwait(false))
{
readings[id] = times;
}
}
var events = await AnalysisQueries.LastEventsAsync(_connection, ids, cancellationToken).ConfigureAwait(false);
foreach (var id in ids)
{
var times = readings.GetValueOrDefault(id) ?? [];
var live = sources.Where(s => s.MeterId == id && IsLive(s)).ToList();
TimeSpan? poll = null;
foreach (var source in live.Where(s => s.SourceType == SourceType.HomeAssistant))
{
if (source.Endpoint is { } endpoint && HaEndpointConfig.Parse(endpoint.Config).UseWebSocket)
{
continue;
}
var interval = TimeSpan.FromMinutes(Math.Max(1, SourceConfig.Parse(source.Config).PollMinutes ?? 60));
poll = poll is { } p && p >= interval ? p : interval;
}
var input = new FreshnessInput(
LastReadingOf(id, times),
events.TryGetValue(id, out var lastEvent) ? lastEvent : null,
times,
live.Count > 0,
poll);
_freshness[id] = FreshnessRules.Evaluate(input, Now);
}
}
/// A source that is expected to deliver on its own (D-18).
private static bool IsLive(MeterSource source) =>
source.SourceType is SourceType.Mqtt or SourceType.Tasmota or SourceType.HomeAssistant;
///
/// The meter's last reading time (A-40): what its stored analysis data recorded, and — for a live meter whose
/// rhythm was read — whatever of the two is later, so a reading ingested since the last recompute still counts.
///
private DateTimeOffset? LastReadingOf(int id, IReadOnlyList times)
{
var stored = _catalog.Meters.TryGetValue(id, out var meter) ? meter.State?.LastReadingAt : null;
var sampled = times.Count > 0 ? times.Max() : (DateTimeOffset?)null;
return stored is { } s && sampled is { } r ? (s >= r ? s : r) : stored ?? sampled;
}
// ---------------------------------------------------------------- physical values
/// A physical meter's sums per bucket of one side, and their total.
private (Tally[] Buckets, Tally Total) Tallies(Side side, LeafData leaf)
{
if (side.Tallies.TryGetValue(leaf.Id, out var cached))
{
return cached;
}
var tallies = side.Buckets.Select(b => leaf.Sum(b.From, b.To, side.Cutoff)).ToArray();
var total = new Tally();
foreach (var tally in tallies)
{
total.Add(tally);
}
side.Tallies[leaf.Id] = (tallies, total);
return (tallies, total);
}
///
/// A physical meter's values on one side: per bucket the sum with the status its coverage gives it (D-14), and the
/// period total with the period's own status. As a contributor () the time outside its
/// service period is a known zero (D-24).
///
private PhysicalValues Physical(Side side, LeafData leaf, bool service)
{
var key = (leaf.Id, service);
if (side.Physical.TryGetValue(key, out var cached))
{
return cached;
}
var (tallies, total) = Tallies(side, leaf);
PhysicalValues result;
if (leaf.Meter.IsPending)
{
result = new PhysicalValues([.. side.Buckets.Select(_ => PendingValue)], PendingValue, null, null);
}
else
{
var runs = service ? leaf.ServiceRuns : leaf.Runs;
var implied = leaf.Meter.Quantity.ImpliedProvenance;
var coverage = CoverageEvaluator.EvaluateSeries(side.Buckets, runs, _zone, [.. tallies.Select(t => t.OpeningBalance)], side.Cutoff);
var values = coverage.Select((c, i) => Withheld(c.ToValue(tallies[i].Amount, tallies[i].ProvenanceWith(implied)), tallies[i])).ToList();
var totalCoverage = CoverageEvaluator.Evaluate(side.PeriodBucket, runs, _zone, total.OpeningBalance, side.Cutoff);
var totalValue = side == _current && _period.HasNotStarted()
? BucketValue.Missing(ValueIssue.NotYetOccurred)
: Withheld(totalCoverage.ToValue(total.Amount, total.ProvenanceWith(implied)), total);
result = new PhysicalValues(values, totalValue, coverage, totalCoverage);
}
side.Physical[key] = result;
return result;
}
///
/// A bucket whose sum left out rows closing after the cutoff (A-05) is not complete, even where coverage reaches its
/// end: a whole rollup day or month is withheld once one of its rows closes after the cutoff — a current-month label
/// beside live readings takes the day's live share with it — so an available status would pass what is shown off
/// as the whole bucket, or an empty day as a true zero (D-14, A-20).
///
private static BucketValue Withheld(BucketValue value, Tally tally) =>
value.Status == BucketStatus.Available && tally.AfterRows > 0
? value with { Status = BucketStatus.Partial, Issue = ValueIssue.RecordedAfterNow }
: value;
// ---------------------------------------------------------------- virtual values
/// A virtual meter evaluated on one side (D-27); null when its definition cannot be evaluated.
private VirtualEvaluation? Evaluate(Side side, int meterId)
{
if (side.Evaluations.TryGetValue(meterId, out var cached))
{
return cached;
}
VirtualEvaluation? evaluation = null;
if (_catalog.Find(meterId) is { IsVirtual: true, Formula: { } formula } meter)
{
var sources = formula.MeterIds.Select(id => SourceFor(side, id)).OfType().ToList();
var kind = meter.Validation?.Kind ?? meter.Quantity.Kind;
evaluation = VirtualEvaluator.Evaluate(meterId, formula, kind, side.Buckets, sources);
}
side.Evaluations[meterId] = evaluation;
return evaluation;
}
/// One source of a formula as the evaluator reads it, with the lifecycle of the source (D-24).
private VirtualSource? SourceFor(Side side, int sourceId)
{
if (_catalog.Find(sourceId) is not { } source)
{
return null;
}
VirtualSource input;
if (source.IsVirtual)
{
input = Evaluate(side, sourceId) is { } nested
? VirtualSource.FromEvaluation(nested)
: _catalog.Graph.CycleFor(sourceId) is { } cycle
? VirtualSource.Failed(sourceId, BucketStatus.Invalid, ValueIssue.DependencyCycle, cycle)
: VirtualSource.Failed(sourceId, BucketStatus.Invalid, ValueIssue.InvalidDefinition, [sourceId]);
}
else
{
input = PhysicalSource(side, _leaves[sourceId]);
}
return input with { InstalledAt = source.Meter.InstalledAt, RetiredAt = source.Meter.RetiredAt };
}
///
/// A physical source for the evaluator (A-12): its figures per local day — covered or not from the coverage
/// evaluator, the resolution and month division of the run covering the day, the day's actual amount — and its
/// own bucket and period states, as a contributor.
///
private VirtualSource PhysicalSource(Side side, LeafData leaf)
{
if (side.Sources.TryGetValue(leaf.Id, out var cached))
{
return cached;
}
VirtualSource source;
if (leaf.Meter.IsPending)
{
source = VirtualSource.Failed(leaf.Id, BucketStatus.Pending, ValueIssue.AnalysisPending, [leaf.Id]);
}
else
{
var windows = side.DayWindows(_zone);
var dayBuckets = windows.Select(w => new AnalysisBucket(w.Day, w.Day.AddDays(1), w.From, w.To, BucketSize.Day)).ToList();
var coverage = CoverageEvaluator.EvaluateSeries(dayBuckets, leaf.Runs, _zone, null, side.Cutoff);
var capped = leaf.CappedRuns(side.Cutoff).Where(r => !r.IsGap).OrderBy(r => r.From).ToList();
var implied = leaf.Meter.Quantity.ImpliedProvenance;
var days = new Dictionary();
var cursor = 0;
for (var i = 0; i < windows.Count; i++)
{
if (coverage[i].Covered <= TimeSpan.Zero)
{
continue;
}
var window = windows[i];
while (cursor < capped.Count && capped[cursor].To <= window.From)
{
cursor++;
}
var divided = true;
var any = false;
for (var r = cursor; r < capped.Count && capped[r].From < window.To; r++)
{
if (capped[r].To > window.From)
{
any = true;
divided &= capped[r].DividedAtMonths;
}
}
var tally = leaf.Sum(window.From, window.To, side.Cutoff);
days[window.Day] = new SourceDay(
tally.Amount, true, coverage[i].Resolution ?? ResolutionClass.Hour, tally.ProvenanceWith(implied), any && divided);
}
var states = Physical(side, leaf, service: true);
source = new VirtualSource(leaf.Id, days) { BucketStates = states.Values, PeriodState = states.Total };
}
side.Sources[leaf.Id] = source;
return source;
}
// ---------------------------------------------------------------- series
private AnalysisSeries MeterSeries(int meterId)
{
var meter = _catalog.Meters[meterId];
var key = SeriesKey.ForMeter(meterId, meter.EnergyTypeId, meter.Quantity.Unit);
var entry = _catalog.Totals.Meters.GetValueOrDefault(meterId);
if (!meter.IsVirtual)
{
var leaf = _leaves[meterId];
var asContributor = _request.AsContributors;
var values = Physical(_current, leaf, service: asContributor);
return new AnalysisSeries(key, meter.Name, SeriesBasis.Physical, meter.Quantity.Kind, meter.Quantity.Unit, values.Values, values.Total, true)
{
Coverage = values.Coverage,
Resolution = values.TotalCoverage?.Resolution,
Availability = AvailableRange.OfRuns(leaf.Runs, Now, _zone),
Freshness = _freshness.GetValueOrDefault(meterId) ?? Freshness.None,
RecordedAfterNow = AfterNowFor([meterId]),
Totals = entry,
Notes = meter.Quantity.Notes,
Comparison = Compare(MeterKey(meterId), side => { var v = Physical(side, leaf, service: asContributor); return (v.Values, v.Total); }, side => MatchedValue(side, meterId)),
};
}
var info = VirtualInfo(meter);
var leaves = LeavesOf(meterId);
var basis = meter.VirtualStatus == VirtualMeterStatus.Legacy ? SeriesBasis.LegacyVirtual : SeriesBasis.Virtual;
var evaluation = Evaluate(_current, meterId);
var series = evaluation is null
? new AnalysisSeries(key, meter.Name, basis, meter.Quantity.Kind, meter.Quantity.Unit, Invalid(meter, _current.Buckets.Count), InvalidValue(meter), false)
: new AnalysisSeries(
key, meter.Name, basis, meter.Quantity.Kind, meter.Quantity.Unit,
MarkLegacy(meter, evaluation.Values), MarkLegacy(meter, evaluation.Total), evaluation.IsAdditive)
{
Resolution = evaluation.Resolution,
Contributions = Contributions(_current, evaluation, [meterId]),
};
return series with
{
Availability = VirtualAvailability(meterId),
Freshness = FreshnessRules.Combine(leaves.Select(l => _freshness.GetValueOrDefault(l) ?? Freshness.None)),
RecordedAfterNow = AfterNowFor(leaves),
Totals = entry,
Virtual = info,
Notes = meter.Quantity.Notes,
Comparison = Compare(MeterKey(meterId), side => VirtualValues(side, meter), side => MatchedValue(side, meterId)),
};
}
private AnalysisSeries MeasureSeries(int energyTypeId, MeasureGroup group)
{
var key = SeriesKey.ForMeasure(energyTypeId, group.Measure, group.Unit);
var (values, total) = MeasureValuesOf(_current, group.MeterIds);
var leaves = group.MeterIds.SelectMany(LeavesOf).Distinct().ToList();
return new AnalysisSeries(key, string.Empty, SeriesBasis.Measure, KindOf(group.Measure), group.Unit, values, total, true)
{
MemberIds = group.MeterIds,
Resolution = MeasureResolution(group.MeterIds),
Availability = AvailableRange.Union(group.MeterIds.Select(MeterAvailability), _zone),
Freshness = FreshnessRules.Combine(leaves.Select(l => _freshness.GetValueOrDefault(l) ?? Freshness.None)),
RecordedAfterNow = AfterNowFor(leaves),
Comparison = Compare(
MeasureKey(energyTypeId, group),
side => MeasureValuesOf(side, group.MeterIds),
side => group.MeterIds.Select(id => MatchedValue(side, id)).Aggregate((double?)0d, (sum, v) => sum is { } s && v is { } x ? s + x : null)),
};
}
///
/// The coarsest resolution among a measure's members (D-51): each as its own series reports it — a physical meter's
/// coverage over the period, a virtual meter's evaluation (divided runs are at most monthly, A-03). A total can be
/// opened no finer than its coarsest member resolves.
///
private ResolutionClass? MeasureResolution(IReadOnlyList members)
{
ResolutionClass? coarsest = null;
foreach (var id in members)
{
var resolution = _catalog.Meters[id].IsVirtual
? Evaluate(_current, id)?.Resolution
: Physical(_current, _leaves[id], service: true).TotalCoverage?.Resolution;
if (resolution is { } value && (coarsest is null || value > coarsest))
{
coarsest = value;
}
}
return coarsest;
}
/// A measure's values on one side: its members' values as contributors, added up (D-22).
private (IReadOnlyList Values, BucketValue Total) MeasureValuesOf(Side side, IReadOnlyList members)
{
var parts = members.Select(id => (id, Values: MemberValues(side, id))).ToList();
var values = MeasureValues.SumSeries([.. parts.Select(p => (p.id, p.Values.Values))], side.Buckets.Count);
var total = MeasureValues.Sum([.. parts.Select(p => (p.id, p.Values.Total))]);
if (side == _current && _period.HasNotStarted())
{
total = BucketValue.Missing(ValueIssue.NotYetOccurred);
}
return (values, total);
}
private (IReadOnlyList Values, BucketValue Total) MemberValues(Side side, int meterId)
{
var meter = _catalog.Meters[meterId];
if (!meter.IsVirtual)
{
var values = Physical(side, _leaves[meterId], service: true);
return (values.Values, values.Total);
}
return VirtualValues(side, meter);
}
private (IReadOnlyList Values, BucketValue Total) VirtualValues(Side side, AnalysisMeter meter)
{
var evaluation = Evaluate(side, meter.Id);
return evaluation is null
? (Invalid(meter, side.Buckets.Count), InvalidValue(meter))
: (MarkLegacy(meter, evaluation.Values), MarkLegacy(meter, evaluation.Total));
}
/// The series on the comparison side, and the change over the coverage both share (D-07, D-08).
private SeriesComparison? Compare(
string key,
Func Values, BucketValue Total)> valuesOn,
Func matchedOn)
{
if (_comparison is null)
{
return null;
}
var (values, total) = valuesOn(_comparison);
var matched = _matched.GetValueOrDefault(key) ?? MatchedCoverageResult.NotComparable;
if (!matched.IsComparable)
{
return new SeriesComparison(values, total, matched, null, null, Change.Unavailable);
}
var current = matchedOn(new Side(MatchedWindows(matched, comparison: false), _current.PeriodBucket, _current.Cutoff));
var previous = matchedOn(new Side(MatchedWindows(matched, comparison: true), _comparison.PeriodBucket, _comparison.Cutoff));
return new SeriesComparison(values, total, matched, current, previous, Change.Between(current, previous));
}
/// The matched pieces of one side as windows to sum.
private static List MatchedWindows(MatchedCoverageResult matched, bool comparison) =>
[.. matched.Pieces.Select(p => comparison ? p.Comparison : p.Current).Select(r => new AnalysisBucket(r.FirstDay, r.LastDay.AddDays(1), r.From, r.To, BucketSize.Day))];
///
/// A meter's value over a side's windows (here: matched pieces): a physical meter's actual sum, a virtual meter's
/// formula applied to its sources' sums — the joint coverage the matched range is (D-27). Null when not evaluable.
///
private double? MatchedValue(Side windows, int meterId)
{
if (_catalog.Find(meterId) is not { } meter)
{
return null;
}
if (!meter.IsVirtual)
{
var leaf = _leaves[meterId];
if (leaf.Meter.IsPending)
{
return null;
}
return windows.Buckets.Sum(w => leaf.Sum(w.From, w.To, windows.Cutoff).Amount);
}
if (meter.Formula is not { } formula)
{
return null;
}
var value = formula.Evaluate(id => MatchedValue(windows, id) ?? double.NaN);
return double.IsFinite(value) ? value : null;
}
private List Contributions(Side side, VirtualEvaluation evaluation, IReadOnlyList path) =>
[.. evaluation.Contributions.Select(c =>
{
var source = _catalog.Find(c.MeterId);
var isVirtual = source?.IsVirtual == true;
IReadOnlyList sourcePath = [.. path, c.MeterId];
var nested = isVirtual && !path.Contains(c.MeterId) && Evaluate(side, c.MeterId) is { } inner
? Contributions(side, inner, sourcePath)
: [];
return new SeriesContribution(
c.MeterId, source?.Name ?? string.Empty, isVirtual, c.Coefficient, c.Values, c.UsedAmounts, c.Total, c.UsedTotal, sourcePath, nested);
})];
private VirtualSeriesInfo VirtualInfo(AnalysisMeter meter)
{
var definition = meter.Definition;
return new VirtualSeriesInfo(
meter.VirtualStatus ?? VirtualMeterStatus.NeedsConfiguration,
definition?.Expression,
meter.CostRule,
meter.Formula?.MeterIds ?? definition?.ReferencedMeterIds ?? [],
LeavesOf(meter.Id),
meter.Validation?.Problems ?? [],
meter.Legacy,
meter.VirtualStatus == VirtualMeterStatus.Malformed ? meter.StoredDefinition?.Problem : null);
}
/// A virtual meter's own buckets when it cannot be evaluated: invalid, naming why (D-26, D-28).
private IReadOnlyList Invalid(AnalysisMeter meter, int count)
{
var value = InvalidValue(meter);
return [.. Enumerable.Repeat(value, count)];
}
private BucketValue InvalidValue(AnalysisMeter meter)
{
var id = meter.Id;
if (_catalog.Graph.CycleFor(id) is { } cycle)
{
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, cycle);
}
var problem = meter.Validation?.Problems.FirstOrDefault();
if (problem is { Kind: VirtualProblemKind.DependencyCycle })
{
IReadOnlyList path = problem.MeterIds.Count > 0 && problem.MeterIds[0] == id ? problem.MeterIds : [id, .. problem.MeterIds];
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, path);
}
if (meter.Legacy is { Outcome: LegacyDerivationOutcome.Cycle } legacy)
{
IReadOnlyList path = legacy.MeterIds.Count > 0 && legacy.MeterIds[0] == id ? legacy.MeterIds : [id, .. legacy.MeterIds];
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, path);
}
IReadOnlyList cause = problem is { MeterIds.Count: > 0 } && problem.MeterIds[0] != id ? [id, problem.MeterIds[0]] : [id];
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.InvalidDefinition, null, cause);
}
/// A legacy meter's evaluated values carry "legacy — confirm" where nothing more important is said (D-28).
private static IReadOnlyList MarkLegacy(AnalysisMeter meter, IReadOnlyList values) =>
meter.VirtualStatus == VirtualMeterStatus.Legacy ? [.. values.Select(v => MarkLegacy(meter, v))] : values;
private static BucketValue MarkLegacy(AnalysisMeter meter, BucketValue value) =>
meter.VirtualStatus == VirtualMeterStatus.Legacy && value.Issue == ValueIssue.None
? value with { Issue = ValueIssue.LegacyDefinition }
: value;
private static QuantityKind KindOf(TotalsMeasure measure) => measure switch
{
TotalsMeasure.Generation => QuantityKind.Generation,
TotalsMeasure.Export => QuantityKind.Export,
TotalsMeasure.Runtime => QuantityKind.Runtime,
_ => QuantityKind.Consumption,
};
// ---------------------------------------------------------------- recorded after now, availability, problems
///
/// A physical meter's rows in the requested range that close after now (D-04, A-05): rollups left out of the
/// current sums, today's rows after now, and the days after today up to the end of the named range.
///
private RecordedAfterNow? AfterNowOf(LeafData leaf)
{
var (_, total) = Tallies(_current, leaf);
var block = new Tally();
block.Add(total);
var rows = block.AfterRows;
var amount = block.AfterAmount;
var first = block.AfterFirstDay;
var last = block.AfterLastDay;
void Add(int count, double sum, DateOnly day)
{
if (count <= 0)
{
return;
}
rows += count;
amount += sum;
first = first is { } f && f <= day ? f : day;
last = last is { } l && l >= day ? l : day;
}
if (_readsToday)
{
var today = RangeParts.LocalDate(Now, _zone);
var later = leaf.RawFrom(today, Now).ToList();
Add(later.Count, later.Sum(r => r.Amount), today);
}
foreach (var (day, rollup) in leaf.AfterNowDays)
{
Add(rollup.Rows, rollup.Amount, day);
}
return rows > 0 && first is { } firstDay && last is { } lastDay ? new RecordedAfterNow(leaf.Id, rows, amount, firstDay, lastDay) : null;
}
private IReadOnlyList AfterNowFor(IEnumerable leaves) =>
[.. leaves.Select(id => _afterNow.GetValueOrDefault(id)).OfType()];
/// A meter's data range (D-19): a physical meter's coverage, a virtual meter's joint coverage.
private AvailableRange? MeterAvailability(int meterId)
{
if (_catalog.Find(meterId) is not { } meter)
{
return null;
}
return meter.IsVirtual
? VirtualAvailability(meterId)
: _leaves.TryGetValue(meterId, out var leaf) ? AvailableRange.OfRuns(leaf.Runs, Now, _zone) : null;
}
///
/// Where every source of a virtual meter has data (D-27): the intersection of their coverage, with the time outside
/// a source's service period counted as its known zero (D-24), within the outer bounds of the sources' own data.
///
private AvailableRange? VirtualAvailability(int meterId)
{
var leaves = LeavesOf(meterId).Where(_leaves.ContainsKey).Select(id => _leaves[id]).ToList();
if (leaves.Count == 0)
{
return null;
}
IReadOnlyList> capped = [.. leaves.Select(l => CoverageRuns.CapAt(l.ServiceRuns, Now, _zone))];
var joint = CoverageRuns.Covered(CoverageRuns.Intersect(capped));
var hull = AvailableRange.Union(leaves.Select(l => AvailableRange.OfRuns(l.Runs, Now, _zone)), _zone);
if (joint is not { First: { } from, Last: { } to } || hull is null)
{
return null;
}
return AvailableRange.Of(from > hull.From ? from : hull.From, to < hull.To ? to : hull.To, _zone);
}
/// The scope's availability (D-19): its meters' data, and for cost its billed meters' data plus manual costs.
private async Task AvailabilityAsync(CancellationToken cancellationToken)
{
var quantity = AvailableRange.Union(ScopeMeters().Select(MeterAvailability), _zone);
if (_request.QuantitiesOnly)
{
return new ScopeAvailability(quantity, null, null);
}
var metered = AvailableRange.Union(CostMeters().Select(MeterAvailability), _zone);
var today = RangeParts.LocalDate(Now, _zone);
var manual = await _db.ManualCosts.AsNoTracking()
.Where(c => c.PeriodStart <= today)
.Select(c => new { c.MeterId, c.PeriodStart })
.ToListAsync(cancellationToken).ConfigureAwait(false);
var scopeMeters = ScopeMeters().ToHashSet();
var days = manual
.Where(c => _request.Scope.Kind == AnalysisScopeKind.Portfolio || (c.MeterId is { } id && scopeMeters.Contains(id)))
.Select(c => c.PeriodStart)
.ToList();
AvailableRange? manualRange = null;
if (days.Count > 0)
{
var from = Core.Normalization.GapAttribution.LocalMidnight(days.Min(), _zone);
var to = Core.Normalization.GapAttribution.LocalMidnight(days.Max().AddDays(1), _zone);
manualRange = AvailableRange.Of(from, to < Now ? to : Now, _zone) ?? AvailableRange.Of(from, to, _zone);
}
var cost = AvailableRange.Union([metered, manualRange], _zone);
LatestPeriod? latest = null;
if (cost is not null)
{
var month = cost.LatestMonth;
var byMeters = metered?.LatestMonth == month;
var byManual = manualRange?.LatestMonth == month;
latest = new LatestPeriod(month, byMeters && byManual ? LatestPeriodBasis.Both : byManual ? LatestPeriodBasis.Manual : LatestPeriodBasis.Meters);
}
return new ScopeAvailability(quantity, cost, latest);
}
/// The attention items (D-53) of everything the result shows.
private void CollectProblems()
{
var shown = _seriesIds.Concat(_measures.SelectMany(m => m.Group.MeterIds)).Distinct().ToList();
var virtuals = new SortedSet();
foreach (var id in shown)
{
CollectVirtuals(id, virtuals);
}
foreach (var id in virtuals)
{
var meter = _catalog.Meters[id];
switch (meter.VirtualStatus)
{
case VirtualMeterStatus.Legacy:
_problems.Add(new AnalysisProblem(AnalysisProblemKind.LegacyDefinition, id)
{
MeterIds = meter.Legacy?.MeterIds ?? [],
Legacy = meter.Legacy?.Outcome,
});
break;
case VirtualMeterStatus.NeedsConfiguration:
_problems.Add(new AnalysisProblem(AnalysisProblemKind.LegacyNeedsConfiguration, id)
{
MeterIds = meter.Legacy?.MeterIds ?? [],
Values = meter.Legacy?.Values ?? [],
Legacy = meter.Legacy?.Outcome,
});
break;
case VirtualMeterStatus.Malformed:
_problems.Add(new AnalysisProblem(AnalysisProblemKind.MalformedDefinition, id)
{
Values = meter.StoredDefinition?.Problem is { } malformation ? [malformation] : [],
});
break;
case VirtualMeterStatus.Invalid:
foreach (var problem in meter.Validation?.Problems ?? [])
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.InvalidDefinition, id)
{
MeterIds = problem.MeterIds,
Values = problem.Values,
Virtual = problem,
});
}
break;
}
}
foreach (var id in _dataLeaves.Order())
{
if (_leaves[id].Meter.IsPending)
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.AnalysisPending, id));
}
if (_afterNow.GetValueOrDefault(id) is { } block)
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.RecordedAfterNow, id) { AfterNow = block });
}
if (_freshness.GetValueOrDefault(id) is { State: FreshnessState.Stale })
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.StaleSource, id));
}
}
if (_request.Scope.Kind == AnalysisScopeKind.Meters)
{
return;
}
var types = TypesInScope();
foreach (var problem in _catalog.Totals.Problems.Where(p => _catalog.Find(p.MeterId) is { } m && types.Contains(m.EnergyTypeId)))
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.TotalsProblem, problem.MeterId)
{
MeterIds = problem.OtherMeterId is { } other ? [other] : [],
Totals = problem,
});
}
foreach (var hint in _catalog.Totals.Hints.Where(h => types.Contains(h.EnergyTypeId)))
{
_problems.Add(new AnalysisProblem(AnalysisProblemKind.PossibleOverlap, hint.MeterId) { MeterIds = [hint.OtherMeterId], Hint = hint });
}
}
private void CollectVirtuals(int meterId, SortedSet found)
{
if (_catalog.Find(meterId) is not { IsVirtual: true } meter || !found.Add(meterId))
{
return;
}
foreach (var source in meter.Formula?.MeterIds ?? [])
{
CollectVirtuals(source, found);
}
}
private HashSet TypesInScope() => _request.Scope.Kind == AnalysisScopeKind.EnergyType
? [_request.Scope.EnergyTypeId!.Value]
: [.. _catalog.Meters.Values.Select(m => m.EnergyTypeId)];
private IReadOnlyList Classification()
{
if (_request.Scope.Kind == AnalysisScopeKind.Meters)
{
return [];
}
return [.. ScopeMeters().Order()
.Where(id => _catalog.Totals.Meters.ContainsKey(id))
.Select(id => new MeterClassification(id, _catalog.Meters[id].Name, _catalog.Totals.Meters[id]))];
}
private static List Deduplicated(List problems)
{
var seen = new HashSet<(AnalysisProblemKind, int?, VirtualProblemKind?, TotalsProblemKind?, OverlapHintKind?)>();
return [.. problems.Where(p => seen.Add((p.Kind, p.MeterId, p.Virtual?.Kind, p.Totals?.Kind, p.Hint?.Kind)))];
}
private static string MeterKey(int meterId) => SeriesKey.ForMeter(meterId, 0, string.Empty).Id;
private static string MeasureKey(int energyTypeId, MeasureGroup group) => SeriesKey.ForMeasure(energyTypeId, group.Measure, group.Unit).Id;
/// The values of one physical meter on one side.
private sealed record PhysicalValues(
IReadOnlyList Values, BucketValue Total, IReadOnlyList? Coverage, BucketCoverage? TotalCoverage);
///
/// One side of a request — the current period or its comparison — with its buckets, the bucket that is the whole
/// period, the cut-off its actuals stop at, and what has been computed for it.
///
private sealed class Side(IReadOnlyList buckets, AnalysisBucket periodBucket, DateTimeOffset cutoff)
{
private List? _dayWindows;
public IReadOnlyList Buckets { get; } = buckets;
public AnalysisBucket PeriodBucket { get; } = periodBucket;
public DateTimeOffset Cutoff { get; } = cutoff;
public Dictionary Tallies { get; } = [];
public Dictionary<(int, bool), PhysicalValues> Physical { get; } = [];
public Dictionary Evaluations { get; } = [];
public Dictionary Sources { get; } = [];
/// The local days the buckets span, each clipped to the buckets' range.
public List DayWindows(TimeZoneInfo zone)
{
if (_dayWindows is not null)
{
return _dayWindows;
}
_dayWindows = [];
if (Buckets.Count == 0)
{
return _dayWindows;
}
var from = Buckets[0].From;
var to = Buckets[^1].To;
if (to <= from)
{
return _dayWindows;
}
for (var day = RangeParts.LocalDate(from, zone); ; day = day.AddDays(1))
{
var start = Core.Normalization.GapAttribution.LocalMidnight(day, zone);
if (start >= to)
{
break;
}
var end = Core.Normalization.GapAttribution.LocalMidnight(day.AddDays(1), zone);
var windowFrom = start > from ? start : from;
var windowTo = end < to ? end : to;
if (windowTo > windowFrom)
{
_dayWindows.Add(new DayWindow(day, windowFrom, windowTo));
}
}
return _dayWindows;
}
}
/// One local day of a side, clipped to its range.
private readonly record struct DayWindow(DateOnly Day, DateTimeOffset From, DateTimeOffset To);
}