Florian Schmidt
a08e9f781f
ci / build-test (push) Successful in 2m41s
Three things a reported Heizoel page got wrong at once. Its tank is dipped a few times a year and its burner read every few months, which is exactly the shape the coverage rules had not been walked through. "No data" for data that exists. A tank books nothing until the next dipstick closes the interval, so the stretch after the last dipstick is covered by no run at all, and a bucket no run covers was reported missing. The burner, whose run reaches into the window, said "only coarser data" -- the honest answer -- so one card claimed there was nothing while the coverage panel beside it listed years of data. A bucket that no run covers, no gap overlaps and no opening balance explains now reports the meter's resolution when its preceding coverage is within one interval of its own class: it is not silent, it is read rarely. A meter that does book its own buckets and stops -- a dead hourly source, a sheet asked about a later month -- still reads missing. Auto answering twelve months with one bar. Coarse only means "longer than a month", so a dipstick taken each autumn straddles a New Year as surely as a month start: coarsening the chart to years bought nothing and cost every point. The planning resolution now caps coarse at month when a run crosses a local year edge, and a series that cannot resolve the natural size no longer coarsens the whole chart -- it is drawn at that size with its buckets marked, which the chart and table already explain. A page contradicting itself. The comparison line above the ranking was fed the leading measure's matched coverage but worded as if it spoke for the page, directly above a burner row that did compare. It now names the figure it is about. Alongside: the "largest changes" ranking no longer drops a meter whose change is not comparable. It ranks what can be ranked, then lists the rest with their values and the reason -- the tank had been vanishing from its own energy type. And every dated table now reads newest first, as lists are read; charts stay chronological left to right, and the CSV export stays ascending for spreadsheets. A-41 to A-43 in the note record the three rules.
1312 lines
55 KiB
C#
1312 lines
55 KiB
C#
using System.Data.Common;
|
|
using MeterVault.Core.Analysis;
|
|
using MeterVault.Core.Analysis.Coverage;
|
|
using MeterVault.Core.Analysis.Totals;
|
|
using MeterVault.Core.Analysis.Virtual;
|
|
using MeterVault.Core.Domain;
|
|
using MeterVault.Infrastructure.Ingestion;
|
|
using MeterVault.Infrastructure.Persistence;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace MeterVault.Infrastructure.Analysis;
|
|
|
|
/// <summary>
|
|
/// One analysis request from catalog to result: which series and meters it needs, what to read for them, and how the
|
|
/// read data becomes bucket values (D-14, D-27), totals (D-22), comparisons (D-07) and availability (D-19).
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The order matters and is fixed: targets → physical leaves → coverage → bucket plan and comparison → matched
|
|
/// coverage → every window a sum will need is registered → the rollups and edge rows are loaded, once per table →
|
|
/// values are computed from memory. Registering before loading is what keeps the reader to one query per table while
|
|
/// still summing arbitrary ranges (a matched range can end mid-day).
|
|
/// </remarks>
|
|
internal sealed class AnalysisRun
|
|
{
|
|
private static readonly BucketValue PendingValue = new(null, BucketStatus.Pending, Provenance.None, ValueIssue.AnalysisPending);
|
|
|
|
private readonly MeterVaultDbContext _db;
|
|
private readonly DbConnection _connection;
|
|
private readonly AnalysisCatalog _catalog;
|
|
private readonly AnalysisRequest _request;
|
|
private readonly ResolvedPeriod _period;
|
|
private readonly TimeZoneInfo _zone;
|
|
|
|
/// <summary>Every physical meter whose coverage is read (availability needs the whole scope).</summary>
|
|
private readonly Dictionary<int, LeafData> _leaves = [];
|
|
|
|
/// <summary>The physical meters whose rollups are read (series, members and virtual sources).</summary>
|
|
private readonly HashSet<int> _dataLeaves = [];
|
|
|
|
private readonly Dictionary<int, IReadOnlyList<int>> _leavesOf = [];
|
|
private readonly Dictionary<int, Freshness> _freshness = [];
|
|
private readonly Dictionary<int, RecordedAfterNow> _afterNow = [];
|
|
private readonly List<AnalysisProblem> _problems = [];
|
|
|
|
private List<int> _seriesIds = [];
|
|
private List<(int EnergyTypeId, MeasureGroup Group)> _measures = [];
|
|
private Side _current = null!;
|
|
private Side? _comparison;
|
|
private ComparisonResolution? _resolution;
|
|
private ResolvedPeriod? _comparisonPeriod;
|
|
private IReadOnlyList<BucketPair> _pairs = [];
|
|
private readonly Dictionary<string, MatchedCoverageResult> _matched = [];
|
|
private bool _readsToday;
|
|
|
|
public AnalysisRun(MeterVaultDbContext db, AnalysisCatalog catalog, AnalysisRequest request, TimeZoneInfo zone)
|
|
{
|
|
_db = db;
|
|
_connection = db.Database.GetDbConnection();
|
|
_catalog = catalog;
|
|
_request = request;
|
|
_period = request.Period;
|
|
_zone = zone;
|
|
}
|
|
|
|
private DateTimeOffset Now => _period.Now;
|
|
|
|
/// <summary>The physical meters this run read, with what it registered and loaded for each (for tests of the D-15 budget).</summary>
|
|
internal IReadOnlyDictionary<int, LeafData> Leaves => _leaves;
|
|
|
|
public async Task<AnalysisResult> ExecuteAsync(BucketPlan? plan, CancellationToken cancellationToken)
|
|
{
|
|
ChooseTargets();
|
|
CollectLeaves(includeScope: true);
|
|
await LoadCoverageAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
plan ??= BucketPlanner.Plan(_period, BucketSize.Auto, CoarsestNeeded(), _request.MaxPoints);
|
|
if (plan.Refused)
|
|
{
|
|
return new AnalysisResult(_request, plan, [], [], ScopeAvailability.None, []) { Refusal = AnalysisRefusal.TooManyPoints };
|
|
}
|
|
|
|
foreach (var id in _dataLeaves)
|
|
{
|
|
_leaves[id].UseMonths = plan.Size is BucketSize.Month or BucketSize.Year && !_leaves[id].IsVirtualSource;
|
|
}
|
|
|
|
_current = new Side(plan.Buckets, PeriodBucket.Of(_period), Now);
|
|
await ResolveComparisonAsync(plan, cancellationToken).ConfigureAwait(false);
|
|
MatchCoverage();
|
|
RegisterWindows();
|
|
await LoadDataAsync(cancellationToken).ConfigureAwait(false);
|
|
if (!_request.QuantitiesOnly)
|
|
{
|
|
await LoadFreshnessAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
foreach (var id in _dataLeaves)
|
|
{
|
|
if (AfterNowOf(_leaves[id]) is { } block)
|
|
{
|
|
_afterNow[id] = block;
|
|
}
|
|
}
|
|
|
|
var series = _seriesIds.Select(MeterSeries).ToList();
|
|
var measures = _measures.Select(m => MeasureSeries(m.EnergyTypeId, m.Group)).ToList();
|
|
var availability = await AvailabilityAsync(cancellationToken).ConfigureAwait(false);
|
|
CollectProblems();
|
|
|
|
return new AnalysisResult(_request, plan, series, measures, availability, Deduplicated(_problems))
|
|
{
|
|
Comparison = _resolution is null ? null : new AnalysisComparison(_resolution, _comparisonPeriod, _pairs),
|
|
Classification = Classification(),
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// The buckets <see cref="BucketSize.Auto"/> chooses for the request (D-05) from the coverage of the data it reads —
|
|
/// the plan <see cref="ExecuteAsync"/> would make, without reading any rollups.
|
|
/// </summary>
|
|
public async Task<BucketPlan> PlanAsync(CancellationToken cancellationToken)
|
|
{
|
|
ChooseTargets();
|
|
CollectLeaves(includeScope: false);
|
|
await LoadCoverageAsync(cancellationToken).ConfigureAwait(false);
|
|
return BucketPlanner.Plan(_period, BucketSize.Auto, CoarsestNeeded(), _request.MaxPoints);
|
|
}
|
|
|
|
public async Task<ScopeAvailability> AvailabilityOnlyAsync(CancellationToken cancellationToken)
|
|
{
|
|
CollectLeaves(includeScope: true);
|
|
await LoadCoverageAsync(cancellationToken).ConfigureAwait(false);
|
|
return await AvailabilityAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- targets and leaves
|
|
|
|
/// <summary>The meter series and measure totals the scope asks for.</summary>
|
|
private void ChooseTargets()
|
|
{
|
|
var scope = _request.Scope;
|
|
switch (scope.Kind)
|
|
{
|
|
case AnalysisScopeKind.Meters:
|
|
foreach (var id in scope.MeterIds)
|
|
{
|
|
if (_catalog.Find(id) is null)
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.UnknownMeter, id));
|
|
}
|
|
else
|
|
{
|
|
_seriesIds.Add(id);
|
|
}
|
|
}
|
|
|
|
break;
|
|
|
|
case AnalysisScopeKind.EnergyType:
|
|
var type = scope.EnergyTypeId!.Value;
|
|
_measures = [.. _catalog.Totals.ForType(type).Measures.Select(g => (type, g))];
|
|
if (_request.IncludeMeterSeries)
|
|
{
|
|
_seriesIds = [.. ScopeMeters().OrderBy(id => id)];
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
_measures = [.. _catalog.Totals.Types.OrderBy(t => t.Key).SelectMany(t => t.Value.Measures.Select(g => (t.Key, g)))];
|
|
if (_request.IncludeMeterSeries)
|
|
{
|
|
_seriesIds = [.. ScopeMeters().OrderBy(id => id)];
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
/// <summary>The meters of the scope: the selection, the energy type's meters, or every meter.</summary>
|
|
private IEnumerable<int> ScopeMeters() => _request.Scope.Kind switch
|
|
{
|
|
AnalysisScopeKind.Meters => _request.Scope.MeterIds.Where(id => _catalog.Find(id) is not null),
|
|
AnalysisScopeKind.EnergyType => _catalog.Meters.Values.Where(m => m.EnergyTypeId == _request.Scope.EnergyTypeId).Select(m => m.Id),
|
|
_ => _catalog.Meters.Keys,
|
|
};
|
|
|
|
/// <summary>The meters whose cost the scope is about (D-19): the selection, or the billed meters of the type(s).</summary>
|
|
private IEnumerable<int> CostMeters() => _request.Scope.Kind switch
|
|
{
|
|
AnalysisScopeKind.Meters => ScopeMeters(),
|
|
AnalysisScopeKind.EnergyType => _catalog.Totals.ForType(_request.Scope.EnergyTypeId!.Value).Billing.Items,
|
|
_ => _catalog.Totals.BillItems,
|
|
};
|
|
|
|
/// <summary>
|
|
/// Expands the targets to the physical meters they read, through evaluable virtual definitions only. Data leaves
|
|
/// feed values; the scope's and the bill's other meters only need their coverage, for availability.
|
|
/// </summary>
|
|
private void CollectLeaves(bool includeScope)
|
|
{
|
|
var dataTargets = _seriesIds.Concat(_measures.SelectMany(m => m.Group.MeterIds)).Distinct().ToList();
|
|
foreach (var id in dataTargets)
|
|
{
|
|
foreach (var leaf in LeavesOf(id))
|
|
{
|
|
_dataLeaves.Add(leaf);
|
|
Leaf(leaf);
|
|
}
|
|
|
|
MarkVirtualSources(id, []);
|
|
}
|
|
|
|
if (!includeScope)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var id in ScopeMeters().Concat(CostMeters()).Distinct())
|
|
{
|
|
foreach (var leaf in LeavesOf(id))
|
|
{
|
|
Leaf(leaf);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>The physical meters a meter reads: itself, or the leaves of its evaluable formula; none when not evaluable.</summary>
|
|
private IReadOnlyList<int> LeavesOf(int meterId)
|
|
{
|
|
if (_leavesOf.TryGetValue(meterId, out var cached))
|
|
{
|
|
return cached;
|
|
}
|
|
|
|
var leaves = new SortedSet<int>();
|
|
var seen = new HashSet<int>();
|
|
var pending = new Stack<int>([meterId]);
|
|
while (pending.Count > 0)
|
|
{
|
|
var id = pending.Pop();
|
|
if (!seen.Add(id) || _catalog.Find(id) is not { } meter)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!meter.IsVirtual)
|
|
{
|
|
leaves.Add(id);
|
|
continue;
|
|
}
|
|
|
|
foreach (var source in meter.Formula?.MeterIds ?? [])
|
|
{
|
|
pending.Push(source);
|
|
}
|
|
}
|
|
|
|
IReadOnlyList<int> result = [.. leaves];
|
|
_leavesOf[meterId] = result;
|
|
return result;
|
|
}
|
|
|
|
/// <summary>Marks the physical meters a virtual meter reads directly or through nested ones: they are read day by day.</summary>
|
|
private void MarkVirtualSources(int meterId, HashSet<int> seen)
|
|
{
|
|
if (!seen.Add(meterId) || _catalog.Find(meterId) is not { IsVirtual: true, Formula: { } formula })
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var source in formula.MeterIds)
|
|
{
|
|
if (_catalog.Find(source) is { IsVirtual: false })
|
|
{
|
|
Leaf(source).IsVirtualSource = true;
|
|
}
|
|
else
|
|
{
|
|
MarkVirtualSources(source, seen);
|
|
}
|
|
}
|
|
}
|
|
|
|
private LeafData Leaf(int meterId)
|
|
{
|
|
if (!_leaves.TryGetValue(meterId, out var leaf))
|
|
{
|
|
_leaves[meterId] = leaf = new LeafData(_catalog.Meters[meterId], _zone);
|
|
}
|
|
|
|
return leaf;
|
|
}
|
|
|
|
private async Task LoadCoverageAsync(CancellationToken cancellationToken)
|
|
{
|
|
var runs = await AnalysisQueries.CoverageAsync(_connection, _leaves.Keys, cancellationToken).ConfigureAwait(false);
|
|
foreach (var (id, leaf) in _leaves)
|
|
{
|
|
leaf.Runs = runs.GetValueOrDefault(id) ?? [];
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// The coarsest resolution among the data covering the period (D-05): auto never plans finer. What each run asks
|
|
/// for is <see cref="ResolutionClassifier.PlanningResolution"/> — a run divided at month boundaries resolves
|
|
/// months whatever its class (A-03), and a run coarser than a month only pushes the chart to years when years
|
|
/// can place it (A-41).
|
|
/// </summary>
|
|
private ResolutionClass? CoarsestNeeded() =>
|
|
ResolutionClassifier.PlanningResolution(
|
|
_dataLeaves
|
|
.SelectMany(id => _leaves[id].CappedRuns(Now))
|
|
.Where(run => !run.IsGap && run.From < _period.To && run.To > _period.From),
|
|
_zone);
|
|
|
|
// ---------------------------------------------------------------- comparison and matched coverage
|
|
|
|
private async Task ResolveComparisonAsync(BucketPlan plan, CancellationToken cancellationToken)
|
|
{
|
|
if (_request.Comparison.Kind == ComparisonKind.None)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_resolution = ComparisonResolver.Resolve(_period, _request.Comparison);
|
|
if (!_resolution.IsApplicable)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var comparison = _resolution.Period;
|
|
_comparisonPeriod = comparison.ToResolvedPeriod(_period);
|
|
_pairs = ComparisonResolver.PairBuckets(_period, comparison, plan.Buckets);
|
|
_comparison = new Side([.. _pairs.Select(p => p.Comparison)], PeriodBucket.Of(_comparisonPeriod), _comparisonPeriod.Now);
|
|
|
|
var stamps = await AnalysisQueries.OpeningBalancesAsync(_connection, _dataLeaves, cancellationToken).ConfigureAwait(false);
|
|
foreach (var (id, stamp) in stamps)
|
|
{
|
|
_leaves[id].OpeningBalanceStamp = stamp;
|
|
}
|
|
}
|
|
|
|
/// <summary>The coverage both periods share (D-07), per series: the meter's own, or all its sources jointly.</summary>
|
|
private void MatchCoverage()
|
|
{
|
|
if (_comparison is null || _resolution?.Period is not { } comparison)
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var id in _seriesIds)
|
|
{
|
|
var meter = _catalog.Meters[id];
|
|
var leaves = LeavesOf(id).Select(l => _leaves[l]).ToList();
|
|
var own = !meter.IsVirtual && !_request.AsContributors;
|
|
_matched[MeterKey(id)] = Match(comparison, leaves, own);
|
|
}
|
|
|
|
foreach (var (type, group) in _measures)
|
|
{
|
|
var leaves = group.MeterIds.SelectMany(LeavesOf).Distinct().Select(l => _leaves[l]).ToList();
|
|
_matched[MeasureKey(type, group)] = Match(comparison, leaves, own: false);
|
|
}
|
|
}
|
|
|
|
private MatchedCoverageResult Match(ComparisonPeriod comparison, List<LeafData> leaves, bool own)
|
|
{
|
|
if (leaves.Count == 0 || leaves.Exists(l => l.Meter.IsPending))
|
|
{
|
|
return MatchedCoverageResult.NotComparable;
|
|
}
|
|
|
|
IReadOnlyList<IReadOnlyList<CoverageRun>> sources = [.. leaves.Select(l => own ? l.Runs : l.ServiceRuns)];
|
|
IReadOnlyList<DateTimeOffset> balances = [.. leaves.Where(l => l.OpeningBalanceStamp is not null).Select(l => l.OpeningBalanceStamp!.Value)];
|
|
var current = MatchSide.OfSources(_period.From, _period.To, Now, _zone, sources, balances);
|
|
var previous = MatchSide.OfSources(comparison.From, comparison.To, _comparison!.Cutoff, _zone, sources, balances);
|
|
return MatchedCoverage.Match(current, previous, instant => comparison.MapInstant(instant, _zone));
|
|
}
|
|
|
|
// ---------------------------------------------------------------- registering and loading
|
|
|
|
/// <summary>Registers every window a sum will read, so the loads fetch exactly those rows.</summary>
|
|
private void RegisterWindows()
|
|
{
|
|
List<Side> sides = [_current];
|
|
if (_comparison is not null)
|
|
{
|
|
sides.Add(_comparison);
|
|
}
|
|
|
|
foreach (var id in _dataLeaves)
|
|
{
|
|
var leaf = _leaves[id];
|
|
foreach (var side in sides)
|
|
{
|
|
foreach (var bucket in side.Buckets)
|
|
{
|
|
leaf.Request(bucket.From, bucket.To);
|
|
}
|
|
|
|
if (leaf.IsVirtualSource)
|
|
{
|
|
foreach (var window in side.DayWindows(_zone))
|
|
{
|
|
leaf.Request(window.From, window.To);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Matched pieces are only summed: their partial days are summed in the database, not loaded (D-15).
|
|
foreach (var (key, matched) in _matched)
|
|
{
|
|
foreach (var leaf in LeavesOfKey(key))
|
|
{
|
|
foreach (var piece in matched.Pieces)
|
|
{
|
|
leaf.RequestSummed(piece.Current.From, piece.Current.To, _current.Cutoff);
|
|
leaf.RequestSummed(piece.Comparison.From, piece.Comparison.To, _comparison!.Cutoff);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Rows of today that close after now go to the "recorded after now" block (D-04).
|
|
var today = RangeParts.LocalDate(Now, _zone);
|
|
_readsToday = ReachesNow() && _period.FirstDay <= today && (_period.Preset == PeriodPreset.AllHistory || _period.NominalLastDay() >= today);
|
|
if (_readsToday)
|
|
{
|
|
foreach (var id in _dataLeaves)
|
|
{
|
|
_leaves[id].RequestRawDay(today);
|
|
}
|
|
}
|
|
}
|
|
|
|
private IEnumerable<LeafData> LeavesOfKey(string key)
|
|
{
|
|
var ids = key.StartsWith('m')
|
|
? LeavesOf(_seriesIds.First(id => MeterKey(id) == key))
|
|
: _measures.Where(m => MeasureKey(m.EnergyTypeId, m.Group) == key).SelectMany(m => m.Group.MeterIds).SelectMany(LeavesOf).Distinct();
|
|
return ids.Select(id => _leaves[id]);
|
|
}
|
|
|
|
/// <summary>True when the requested range reaches now or lies after it: then rows may close after now (D-04).</summary>
|
|
private bool ReachesNow() => _period.IsToDate || _period.ExtendsPastNow || _period.HasNotStarted();
|
|
|
|
private async Task LoadDataAsync(CancellationToken cancellationToken)
|
|
{
|
|
var data = _dataLeaves.ToDictionary(id => id, id => _leaves[id]);
|
|
var today = RangeParts.LocalDate(Now, _zone);
|
|
var afterFrom = _period.FirstDay > today ? _period.FirstDay : today.AddDays(1);
|
|
DateOnly? afterTo = _period.Preset == PeriodPreset.AllHistory ? null : _period.NominalLastDay().AddDays(1);
|
|
IReadOnlyCollection<int> afterIds = ReachesNow() ? _dataLeaves : [];
|
|
|
|
await AnalysisQueries.DayRollupsAsync(_connection, data, afterIds, afterFrom, afterTo, Now, cancellationToken).ConfigureAwait(false);
|
|
await AnalysisQueries.MonthRollupsAsync(_connection, data, cancellationToken).ConfigureAwait(false);
|
|
await AnalysisQueries.RawDaysAsync(_connection, data, _zone, cancellationToken).ConfigureAwait(false);
|
|
await AnalysisQueries.WindowSumsAsync(_connection, data, cancellationToken).ConfigureAwait(false);
|
|
foreach (var leaf in data.Values)
|
|
{
|
|
leaf.Seal();
|
|
}
|
|
}
|
|
|
|
/// <summary>Freshness of every data leaf (D-18): the last reading and event, and the live sources' rhythm.</summary>
|
|
/// <remarks>
|
|
/// The mark — the meter's last reading — is read from its stored rollup state, which the recompute behind every
|
|
/// write path records (A-40); an import-only meter's mark is years old and must stay exact, so it is never
|
|
/// guessed from a bounded sample. Only the *rhythm* of a live source needs raw reading times, and only from the
|
|
/// recent past: that query carries <see cref="FreshnessRules.RecentWindow"/> as its lower bound, so it plans over
|
|
/// the newest raw chunks instead of every chunk a decade of history has. A live meter that delivered nothing
|
|
/// inside the window has no rhythm there; those few meters are asked again over their whole history, so the
|
|
/// stale verdict of a long-silent source is unchanged.
|
|
/// </remarks>
|
|
private async Task LoadFreshnessAsync(CancellationToken cancellationToken)
|
|
{
|
|
var ids = _dataLeaves.ToList();
|
|
if (ids.Count == 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var sources = await _db.MeterSources.AsNoTracking()
|
|
.Include(s => s.Endpoint)
|
|
.Where(s => ids.Contains(s.MeterId) && s.IsEnabled)
|
|
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
|
|
|
var liveIds = ids.Where(id => sources.Exists(s => s.MeterId == id && IsLive(s))).ToList();
|
|
var readings = await AnalysisQueries
|
|
.RecentReadingsAsync(_connection, liveIds, Now - FreshnessRules.RecentWindow, cancellationToken).ConfigureAwait(false);
|
|
var silent = liveIds.Where(id => (readings.GetValueOrDefault(id)?.Count ?? 0) < 2).ToList();
|
|
if (silent.Count > 0)
|
|
{
|
|
foreach (var (id, times) in await AnalysisQueries
|
|
.RecentReadingsAsync(_connection, silent, since: null, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
readings[id] = times;
|
|
}
|
|
}
|
|
|
|
var events = await AnalysisQueries.LastEventsAsync(_connection, ids, cancellationToken).ConfigureAwait(false);
|
|
|
|
foreach (var id in ids)
|
|
{
|
|
var times = readings.GetValueOrDefault(id) ?? [];
|
|
var live = sources.Where(s => s.MeterId == id && IsLive(s)).ToList();
|
|
TimeSpan? poll = null;
|
|
foreach (var source in live.Where(s => s.SourceType == SourceType.HomeAssistant))
|
|
{
|
|
if (source.Endpoint is { } endpoint && HaEndpointConfig.Parse(endpoint.Config).UseWebSocket)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var interval = TimeSpan.FromMinutes(Math.Max(1, SourceConfig.Parse(source.Config).PollMinutes ?? 60));
|
|
poll = poll is { } p && p >= interval ? p : interval;
|
|
}
|
|
|
|
var input = new FreshnessInput(
|
|
LastReadingOf(id, times),
|
|
events.TryGetValue(id, out var lastEvent) ? lastEvent : null,
|
|
times,
|
|
live.Count > 0,
|
|
poll);
|
|
_freshness[id] = FreshnessRules.Evaluate(input, Now);
|
|
}
|
|
}
|
|
|
|
/// <summary>A source that is expected to deliver on its own (D-18).</summary>
|
|
private static bool IsLive(MeterSource source) =>
|
|
source.SourceType is SourceType.Mqtt or SourceType.Tasmota or SourceType.HomeAssistant;
|
|
|
|
/// <summary>
|
|
/// The meter's last reading time (A-40): what its stored analysis data recorded, and — for a live meter whose
|
|
/// rhythm was read — whatever of the two is later, so a reading ingested since the last recompute still counts.
|
|
/// </summary>
|
|
private DateTimeOffset? LastReadingOf(int id, IReadOnlyList<DateTimeOffset> times)
|
|
{
|
|
var stored = _catalog.Meters.TryGetValue(id, out var meter) ? meter.State?.LastReadingAt : null;
|
|
var sampled = times.Count > 0 ? times.Max() : (DateTimeOffset?)null;
|
|
return stored is { } s && sampled is { } r ? (s >= r ? s : r) : stored ?? sampled;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- physical values
|
|
|
|
/// <summary>A physical meter's sums per bucket of one side, and their total.</summary>
|
|
private (Tally[] Buckets, Tally Total) Tallies(Side side, LeafData leaf)
|
|
{
|
|
if (side.Tallies.TryGetValue(leaf.Id, out var cached))
|
|
{
|
|
return cached;
|
|
}
|
|
|
|
var tallies = side.Buckets.Select(b => leaf.Sum(b.From, b.To, side.Cutoff)).ToArray();
|
|
var total = new Tally();
|
|
foreach (var tally in tallies)
|
|
{
|
|
total.Add(tally);
|
|
}
|
|
|
|
side.Tallies[leaf.Id] = (tallies, total);
|
|
return (tallies, total);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A physical meter's values on one side: per bucket the sum with the status its coverage gives it (D-14), and the
|
|
/// period total with the period's own status. As a contributor (<paramref name="service"/>) the time outside its
|
|
/// service period is a known zero (D-24).
|
|
/// </summary>
|
|
private PhysicalValues Physical(Side side, LeafData leaf, bool service)
|
|
{
|
|
var key = (leaf.Id, service);
|
|
if (side.Physical.TryGetValue(key, out var cached))
|
|
{
|
|
return cached;
|
|
}
|
|
|
|
var (tallies, total) = Tallies(side, leaf);
|
|
PhysicalValues result;
|
|
if (leaf.Meter.IsPending)
|
|
{
|
|
result = new PhysicalValues([.. side.Buckets.Select(_ => PendingValue)], PendingValue, null, null);
|
|
}
|
|
else
|
|
{
|
|
var runs = service ? leaf.ServiceRuns : leaf.Runs;
|
|
var implied = leaf.Meter.Quantity.ImpliedProvenance;
|
|
var coverage = CoverageEvaluator.EvaluateSeries(side.Buckets, runs, _zone, [.. tallies.Select(t => t.OpeningBalance)], side.Cutoff);
|
|
var values = coverage.Select((c, i) => Withheld(c.ToValue(tallies[i].Amount, tallies[i].ProvenanceWith(implied)), tallies[i])).ToList();
|
|
var totalCoverage = CoverageEvaluator.Evaluate(side.PeriodBucket, runs, _zone, total.OpeningBalance, side.Cutoff);
|
|
var totalValue = side == _current && _period.HasNotStarted()
|
|
? BucketValue.Missing(ValueIssue.NotYetOccurred)
|
|
: Withheld(totalCoverage.ToValue(total.Amount, total.ProvenanceWith(implied)), total);
|
|
result = new PhysicalValues(values, totalValue, coverage, totalCoverage);
|
|
}
|
|
|
|
side.Physical[key] = result;
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// A bucket whose sum left out rows closing after the cutoff (A-05) is not complete, even where coverage reaches its
|
|
/// end: a whole rollup day or month is withheld once one of its rows closes after the cutoff — a current-month label
|
|
/// beside live readings takes the day's live share with it — so an available status would pass what is shown off
|
|
/// as the whole bucket, or an empty day as a true zero (D-14, A-20).
|
|
/// </summary>
|
|
private static BucketValue Withheld(BucketValue value, Tally tally) =>
|
|
value.Status == BucketStatus.Available && tally.AfterRows > 0
|
|
? value with { Status = BucketStatus.Partial, Issue = ValueIssue.RecordedAfterNow }
|
|
: value;
|
|
|
|
// ---------------------------------------------------------------- virtual values
|
|
|
|
/// <summary>A virtual meter evaluated on one side (D-27); null when its definition cannot be evaluated.</summary>
|
|
private VirtualEvaluation? Evaluate(Side side, int meterId)
|
|
{
|
|
if (side.Evaluations.TryGetValue(meterId, out var cached))
|
|
{
|
|
return cached;
|
|
}
|
|
|
|
VirtualEvaluation? evaluation = null;
|
|
if (_catalog.Find(meterId) is { IsVirtual: true, Formula: { } formula } meter)
|
|
{
|
|
var sources = formula.MeterIds.Select(id => SourceFor(side, id)).OfType<VirtualSource>().ToList();
|
|
var kind = meter.Validation?.Kind ?? meter.Quantity.Kind;
|
|
evaluation = VirtualEvaluator.Evaluate(meterId, formula, kind, side.Buckets, sources);
|
|
}
|
|
|
|
side.Evaluations[meterId] = evaluation;
|
|
return evaluation;
|
|
}
|
|
|
|
/// <summary>One source of a formula as the evaluator reads it, with the lifecycle of the source (D-24).</summary>
|
|
private VirtualSource? SourceFor(Side side, int sourceId)
|
|
{
|
|
if (_catalog.Find(sourceId) is not { } source)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
VirtualSource input;
|
|
if (source.IsVirtual)
|
|
{
|
|
input = Evaluate(side, sourceId) is { } nested
|
|
? VirtualSource.FromEvaluation(nested)
|
|
: _catalog.Graph.CycleFor(sourceId) is { } cycle
|
|
? VirtualSource.Failed(sourceId, BucketStatus.Invalid, ValueIssue.DependencyCycle, cycle)
|
|
: VirtualSource.Failed(sourceId, BucketStatus.Invalid, ValueIssue.InvalidDefinition, [sourceId]);
|
|
}
|
|
else
|
|
{
|
|
input = PhysicalSource(side, _leaves[sourceId]);
|
|
}
|
|
|
|
return input with { InstalledAt = source.Meter.InstalledAt, RetiredAt = source.Meter.RetiredAt };
|
|
}
|
|
|
|
/// <summary>
|
|
/// A physical source for the evaluator (A-12): its figures per local day — covered or not from the coverage
|
|
/// evaluator, the resolution and month division of the run covering the day, the day's actual amount — and its
|
|
/// own bucket and period states, as a contributor.
|
|
/// </summary>
|
|
private VirtualSource PhysicalSource(Side side, LeafData leaf)
|
|
{
|
|
if (side.Sources.TryGetValue(leaf.Id, out var cached))
|
|
{
|
|
return cached;
|
|
}
|
|
|
|
VirtualSource source;
|
|
if (leaf.Meter.IsPending)
|
|
{
|
|
source = VirtualSource.Failed(leaf.Id, BucketStatus.Pending, ValueIssue.AnalysisPending, [leaf.Id]);
|
|
}
|
|
else
|
|
{
|
|
var windows = side.DayWindows(_zone);
|
|
var dayBuckets = windows.Select(w => new AnalysisBucket(w.Day, w.Day.AddDays(1), w.From, w.To, BucketSize.Day)).ToList();
|
|
var coverage = CoverageEvaluator.EvaluateSeries(dayBuckets, leaf.Runs, _zone, null, side.Cutoff);
|
|
var capped = leaf.CappedRuns(side.Cutoff).Where(r => !r.IsGap).OrderBy(r => r.From).ToList();
|
|
var implied = leaf.Meter.Quantity.ImpliedProvenance;
|
|
|
|
var days = new Dictionary<DateOnly, SourceDay>();
|
|
var cursor = 0;
|
|
for (var i = 0; i < windows.Count; i++)
|
|
{
|
|
if (coverage[i].Covered <= TimeSpan.Zero)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var window = windows[i];
|
|
while (cursor < capped.Count && capped[cursor].To <= window.From)
|
|
{
|
|
cursor++;
|
|
}
|
|
|
|
var divided = true;
|
|
var any = false;
|
|
for (var r = cursor; r < capped.Count && capped[r].From < window.To; r++)
|
|
{
|
|
if (capped[r].To > window.From)
|
|
{
|
|
any = true;
|
|
divided &= capped[r].DividedAtMonths;
|
|
}
|
|
}
|
|
|
|
var tally = leaf.Sum(window.From, window.To, side.Cutoff);
|
|
days[window.Day] = new SourceDay(
|
|
tally.Amount, true, coverage[i].Resolution ?? ResolutionClass.Hour, tally.ProvenanceWith(implied), any && divided);
|
|
}
|
|
|
|
var states = Physical(side, leaf, service: true);
|
|
source = new VirtualSource(leaf.Id, days) { BucketStates = states.Values, PeriodState = states.Total };
|
|
}
|
|
|
|
side.Sources[leaf.Id] = source;
|
|
return source;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- series
|
|
|
|
private AnalysisSeries MeterSeries(int meterId)
|
|
{
|
|
var meter = _catalog.Meters[meterId];
|
|
var key = SeriesKey.ForMeter(meterId, meter.EnergyTypeId, meter.Quantity.Unit);
|
|
var entry = _catalog.Totals.Meters.GetValueOrDefault(meterId);
|
|
|
|
if (!meter.IsVirtual)
|
|
{
|
|
var leaf = _leaves[meterId];
|
|
var asContributor = _request.AsContributors;
|
|
var values = Physical(_current, leaf, service: asContributor);
|
|
return new AnalysisSeries(key, meter.Name, SeriesBasis.Physical, meter.Quantity.Kind, meter.Quantity.Unit, values.Values, values.Total, true)
|
|
{
|
|
Coverage = values.Coverage,
|
|
Resolution = values.TotalCoverage?.Resolution,
|
|
Availability = AvailableRange.OfRuns(leaf.Runs, Now, _zone),
|
|
Freshness = _freshness.GetValueOrDefault(meterId) ?? Freshness.None,
|
|
RecordedAfterNow = AfterNowFor([meterId]),
|
|
Totals = entry,
|
|
Notes = meter.Quantity.Notes,
|
|
Comparison = Compare(MeterKey(meterId), side => { var v = Physical(side, leaf, service: asContributor); return (v.Values, v.Total); }, side => MatchedValue(side, meterId)),
|
|
};
|
|
}
|
|
|
|
var info = VirtualInfo(meter);
|
|
var leaves = LeavesOf(meterId);
|
|
var basis = meter.VirtualStatus == VirtualMeterStatus.Legacy ? SeriesBasis.LegacyVirtual : SeriesBasis.Virtual;
|
|
var evaluation = Evaluate(_current, meterId);
|
|
var series = evaluation is null
|
|
? new AnalysisSeries(key, meter.Name, basis, meter.Quantity.Kind, meter.Quantity.Unit, Invalid(meter, _current.Buckets.Count), InvalidValue(meter), false)
|
|
: new AnalysisSeries(
|
|
key, meter.Name, basis, meter.Quantity.Kind, meter.Quantity.Unit,
|
|
MarkLegacy(meter, evaluation.Values), MarkLegacy(meter, evaluation.Total), evaluation.IsAdditive)
|
|
{
|
|
Resolution = evaluation.Resolution,
|
|
Contributions = Contributions(_current, evaluation, [meterId]),
|
|
};
|
|
|
|
return series with
|
|
{
|
|
Availability = VirtualAvailability(meterId),
|
|
Freshness = FreshnessRules.Combine(leaves.Select(l => _freshness.GetValueOrDefault(l) ?? Freshness.None)),
|
|
RecordedAfterNow = AfterNowFor(leaves),
|
|
Totals = entry,
|
|
Virtual = info,
|
|
Notes = meter.Quantity.Notes,
|
|
Comparison = Compare(MeterKey(meterId), side => VirtualValues(side, meter), side => MatchedValue(side, meterId)),
|
|
};
|
|
}
|
|
|
|
private AnalysisSeries MeasureSeries(int energyTypeId, MeasureGroup group)
|
|
{
|
|
var key = SeriesKey.ForMeasure(energyTypeId, group.Measure, group.Unit);
|
|
var (values, total) = MeasureValuesOf(_current, group.MeterIds);
|
|
var leaves = group.MeterIds.SelectMany(LeavesOf).Distinct().ToList();
|
|
return new AnalysisSeries(key, string.Empty, SeriesBasis.Measure, KindOf(group.Measure), group.Unit, values, total, true)
|
|
{
|
|
MemberIds = group.MeterIds,
|
|
Resolution = MeasureResolution(group.MeterIds),
|
|
Availability = AvailableRange.Union(group.MeterIds.Select(MeterAvailability), _zone),
|
|
Freshness = FreshnessRules.Combine(leaves.Select(l => _freshness.GetValueOrDefault(l) ?? Freshness.None)),
|
|
RecordedAfterNow = AfterNowFor(leaves),
|
|
Comparison = Compare(
|
|
MeasureKey(energyTypeId, group),
|
|
side => MeasureValuesOf(side, group.MeterIds),
|
|
side => group.MeterIds.Select(id => MatchedValue(side, id)).Aggregate((double?)0d, (sum, v) => sum is { } s && v is { } x ? s + x : null)),
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// The coarsest resolution among a measure's members (D-51): each as its own series reports it — a physical meter's
|
|
/// coverage over the period, a virtual meter's evaluation (divided runs are at most monthly, A-03). A total can be
|
|
/// opened no finer than its coarsest member resolves.
|
|
/// </summary>
|
|
private ResolutionClass? MeasureResolution(IReadOnlyList<int> members)
|
|
{
|
|
ResolutionClass? coarsest = null;
|
|
foreach (var id in members)
|
|
{
|
|
var resolution = _catalog.Meters[id].IsVirtual
|
|
? Evaluate(_current, id)?.Resolution
|
|
: Physical(_current, _leaves[id], service: true).TotalCoverage?.Resolution;
|
|
if (resolution is { } value && (coarsest is null || value > coarsest))
|
|
{
|
|
coarsest = value;
|
|
}
|
|
}
|
|
|
|
return coarsest;
|
|
}
|
|
|
|
/// <summary>A measure's values on one side: its members' values as contributors, added up (D-22).</summary>
|
|
private (IReadOnlyList<BucketValue> Values, BucketValue Total) MeasureValuesOf(Side side, IReadOnlyList<int> members)
|
|
{
|
|
var parts = members.Select(id => (id, Values: MemberValues(side, id))).ToList();
|
|
var values = MeasureValues.SumSeries([.. parts.Select(p => (p.id, p.Values.Values))], side.Buckets.Count);
|
|
var total = MeasureValues.Sum([.. parts.Select(p => (p.id, p.Values.Total))]);
|
|
if (side == _current && _period.HasNotStarted())
|
|
{
|
|
total = BucketValue.Missing(ValueIssue.NotYetOccurred);
|
|
}
|
|
|
|
return (values, total);
|
|
}
|
|
|
|
private (IReadOnlyList<BucketValue> Values, BucketValue Total) MemberValues(Side side, int meterId)
|
|
{
|
|
var meter = _catalog.Meters[meterId];
|
|
if (!meter.IsVirtual)
|
|
{
|
|
var values = Physical(side, _leaves[meterId], service: true);
|
|
return (values.Values, values.Total);
|
|
}
|
|
|
|
return VirtualValues(side, meter);
|
|
}
|
|
|
|
private (IReadOnlyList<BucketValue> Values, BucketValue Total) VirtualValues(Side side, AnalysisMeter meter)
|
|
{
|
|
var evaluation = Evaluate(side, meter.Id);
|
|
return evaluation is null
|
|
? (Invalid(meter, side.Buckets.Count), InvalidValue(meter))
|
|
: (MarkLegacy(meter, evaluation.Values), MarkLegacy(meter, evaluation.Total));
|
|
}
|
|
|
|
/// <summary>The series on the comparison side, and the change over the coverage both share (D-07, D-08).</summary>
|
|
private SeriesComparison? Compare(
|
|
string key,
|
|
Func<Side, (IReadOnlyList<BucketValue> Values, BucketValue Total)> valuesOn,
|
|
Func<Side, double?> matchedOn)
|
|
{
|
|
if (_comparison is null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var (values, total) = valuesOn(_comparison);
|
|
var matched = _matched.GetValueOrDefault(key) ?? MatchedCoverageResult.NotComparable;
|
|
if (!matched.IsComparable)
|
|
{
|
|
return new SeriesComparison(values, total, matched, null, null, Change.Unavailable);
|
|
}
|
|
|
|
var current = matchedOn(new Side(MatchedWindows(matched, comparison: false), _current.PeriodBucket, _current.Cutoff));
|
|
var previous = matchedOn(new Side(MatchedWindows(matched, comparison: true), _comparison.PeriodBucket, _comparison.Cutoff));
|
|
return new SeriesComparison(values, total, matched, current, previous, Change.Between(current, previous));
|
|
}
|
|
|
|
/// <summary>The matched pieces of one side as windows to sum.</summary>
|
|
private static List<AnalysisBucket> MatchedWindows(MatchedCoverageResult matched, bool comparison) =>
|
|
[.. matched.Pieces.Select(p => comparison ? p.Comparison : p.Current).Select(r => new AnalysisBucket(r.FirstDay, r.LastDay.AddDays(1), r.From, r.To, BucketSize.Day))];
|
|
|
|
/// <summary>
|
|
/// A meter's value over a side's windows (here: matched pieces): a physical meter's actual sum, a virtual meter's
|
|
/// formula applied to its sources' sums — the joint coverage the matched range is (D-27). Null when not evaluable.
|
|
/// </summary>
|
|
private double? MatchedValue(Side windows, int meterId)
|
|
{
|
|
if (_catalog.Find(meterId) is not { } meter)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
if (!meter.IsVirtual)
|
|
{
|
|
var leaf = _leaves[meterId];
|
|
if (leaf.Meter.IsPending)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return windows.Buckets.Sum(w => leaf.Sum(w.From, w.To, windows.Cutoff).Amount);
|
|
}
|
|
|
|
if (meter.Formula is not { } formula)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
var value = formula.Evaluate(id => MatchedValue(windows, id) ?? double.NaN);
|
|
return double.IsFinite(value) ? value : null;
|
|
}
|
|
|
|
private List<SeriesContribution> Contributions(Side side, VirtualEvaluation evaluation, IReadOnlyList<int> path) =>
|
|
[.. evaluation.Contributions.Select(c =>
|
|
{
|
|
var source = _catalog.Find(c.MeterId);
|
|
var isVirtual = source?.IsVirtual == true;
|
|
IReadOnlyList<int> sourcePath = [.. path, c.MeterId];
|
|
var nested = isVirtual && !path.Contains(c.MeterId) && Evaluate(side, c.MeterId) is { } inner
|
|
? Contributions(side, inner, sourcePath)
|
|
: [];
|
|
return new SeriesContribution(
|
|
c.MeterId, source?.Name ?? string.Empty, isVirtual, c.Coefficient, c.Values, c.UsedAmounts, c.Total, c.UsedTotal, sourcePath, nested);
|
|
})];
|
|
|
|
private VirtualSeriesInfo VirtualInfo(AnalysisMeter meter)
|
|
{
|
|
var definition = meter.Definition;
|
|
return new VirtualSeriesInfo(
|
|
meter.VirtualStatus ?? VirtualMeterStatus.NeedsConfiguration,
|
|
definition?.Expression,
|
|
meter.CostRule,
|
|
meter.Formula?.MeterIds ?? definition?.ReferencedMeterIds ?? [],
|
|
LeavesOf(meter.Id),
|
|
meter.Validation?.Problems ?? [],
|
|
meter.Legacy,
|
|
meter.VirtualStatus == VirtualMeterStatus.Malformed ? meter.StoredDefinition?.Problem : null);
|
|
}
|
|
|
|
/// <summary>A virtual meter's own buckets when it cannot be evaluated: invalid, naming why (D-26, D-28).</summary>
|
|
private IReadOnlyList<BucketValue> Invalid(AnalysisMeter meter, int count)
|
|
{
|
|
var value = InvalidValue(meter);
|
|
return [.. Enumerable.Repeat(value, count)];
|
|
}
|
|
|
|
private BucketValue InvalidValue(AnalysisMeter meter)
|
|
{
|
|
var id = meter.Id;
|
|
if (_catalog.Graph.CycleFor(id) is { } cycle)
|
|
{
|
|
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, cycle);
|
|
}
|
|
|
|
var problem = meter.Validation?.Problems.FirstOrDefault();
|
|
if (problem is { Kind: VirtualProblemKind.DependencyCycle })
|
|
{
|
|
IReadOnlyList<int> path = problem.MeterIds.Count > 0 && problem.MeterIds[0] == id ? problem.MeterIds : [id, .. problem.MeterIds];
|
|
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, path);
|
|
}
|
|
|
|
if (meter.Legacy is { Outcome: LegacyDerivationOutcome.Cycle } legacy)
|
|
{
|
|
IReadOnlyList<int> path = legacy.MeterIds.Count > 0 && legacy.MeterIds[0] == id ? legacy.MeterIds : [id, .. legacy.MeterIds];
|
|
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.DependencyCycle, null, path);
|
|
}
|
|
|
|
IReadOnlyList<int> cause = problem is { MeterIds.Count: > 0 } && problem.MeterIds[0] != id ? [id, problem.MeterIds[0]] : [id];
|
|
return new BucketValue(null, BucketStatus.Invalid, Provenance.Derived, ValueIssue.InvalidDefinition, null, cause);
|
|
}
|
|
|
|
/// <summary>A legacy meter's evaluated values carry "legacy — confirm" where nothing more important is said (D-28).</summary>
|
|
private static IReadOnlyList<BucketValue> MarkLegacy(AnalysisMeter meter, IReadOnlyList<BucketValue> values) =>
|
|
meter.VirtualStatus == VirtualMeterStatus.Legacy ? [.. values.Select(v => MarkLegacy(meter, v))] : values;
|
|
|
|
private static BucketValue MarkLegacy(AnalysisMeter meter, BucketValue value) =>
|
|
meter.VirtualStatus == VirtualMeterStatus.Legacy && value.Issue == ValueIssue.None
|
|
? value with { Issue = ValueIssue.LegacyDefinition }
|
|
: value;
|
|
|
|
private static QuantityKind KindOf(TotalsMeasure measure) => measure switch
|
|
{
|
|
TotalsMeasure.Generation => QuantityKind.Generation,
|
|
TotalsMeasure.Export => QuantityKind.Export,
|
|
TotalsMeasure.Runtime => QuantityKind.Runtime,
|
|
_ => QuantityKind.Consumption,
|
|
};
|
|
|
|
// ---------------------------------------------------------------- recorded after now, availability, problems
|
|
|
|
/// <summary>
|
|
/// A physical meter's rows in the requested range that close after now (D-04, A-05): rollups left out of the
|
|
/// current sums, today's rows after now, and the days after today up to the end of the named range.
|
|
/// </summary>
|
|
private RecordedAfterNow? AfterNowOf(LeafData leaf)
|
|
{
|
|
var (_, total) = Tallies(_current, leaf);
|
|
var block = new Tally();
|
|
block.Add(total);
|
|
|
|
var rows = block.AfterRows;
|
|
var amount = block.AfterAmount;
|
|
var first = block.AfterFirstDay;
|
|
var last = block.AfterLastDay;
|
|
|
|
void Add(int count, double sum, DateOnly day)
|
|
{
|
|
if (count <= 0)
|
|
{
|
|
return;
|
|
}
|
|
|
|
rows += count;
|
|
amount += sum;
|
|
first = first is { } f && f <= day ? f : day;
|
|
last = last is { } l && l >= day ? l : day;
|
|
}
|
|
|
|
if (_readsToday)
|
|
{
|
|
var today = RangeParts.LocalDate(Now, _zone);
|
|
var later = leaf.RawFrom(today, Now).ToList();
|
|
Add(later.Count, later.Sum(r => r.Amount), today);
|
|
}
|
|
|
|
foreach (var (day, rollup) in leaf.AfterNowDays)
|
|
{
|
|
Add(rollup.Rows, rollup.Amount, day);
|
|
}
|
|
|
|
return rows > 0 && first is { } firstDay && last is { } lastDay ? new RecordedAfterNow(leaf.Id, rows, amount, firstDay, lastDay) : null;
|
|
}
|
|
|
|
private IReadOnlyList<RecordedAfterNow> AfterNowFor(IEnumerable<int> leaves) =>
|
|
[.. leaves.Select(id => _afterNow.GetValueOrDefault(id)).OfType<RecordedAfterNow>()];
|
|
|
|
/// <summary>A meter's data range (D-19): a physical meter's coverage, a virtual meter's joint coverage.</summary>
|
|
private AvailableRange? MeterAvailability(int meterId)
|
|
{
|
|
if (_catalog.Find(meterId) is not { } meter)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return meter.IsVirtual
|
|
? VirtualAvailability(meterId)
|
|
: _leaves.TryGetValue(meterId, out var leaf) ? AvailableRange.OfRuns(leaf.Runs, Now, _zone) : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Where every source of a virtual meter has data (D-27): the intersection of their coverage, with the time outside
|
|
/// a source's service period counted as its known zero (D-24), within the outer bounds of the sources' own data.
|
|
/// </summary>
|
|
private AvailableRange? VirtualAvailability(int meterId)
|
|
{
|
|
var leaves = LeavesOf(meterId).Where(_leaves.ContainsKey).Select(id => _leaves[id]).ToList();
|
|
if (leaves.Count == 0)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
IReadOnlyList<IReadOnlyList<CoverageRun>> capped = [.. leaves.Select(l => CoverageRuns.CapAt(l.ServiceRuns, Now, _zone))];
|
|
var joint = CoverageRuns.Covered(CoverageRuns.Intersect(capped));
|
|
var hull = AvailableRange.Union(leaves.Select(l => AvailableRange.OfRuns(l.Runs, Now, _zone)), _zone);
|
|
if (joint is not { First: { } from, Last: { } to } || hull is null)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
return AvailableRange.Of(from > hull.From ? from : hull.From, to < hull.To ? to : hull.To, _zone);
|
|
}
|
|
|
|
/// <summary>The scope's availability (D-19): its meters' data, and for cost its billed meters' data plus manual costs.</summary>
|
|
private async Task<ScopeAvailability> AvailabilityAsync(CancellationToken cancellationToken)
|
|
{
|
|
var quantity = AvailableRange.Union(ScopeMeters().Select(MeterAvailability), _zone);
|
|
if (_request.QuantitiesOnly)
|
|
{
|
|
return new ScopeAvailability(quantity, null, null);
|
|
}
|
|
|
|
var metered = AvailableRange.Union(CostMeters().Select(MeterAvailability), _zone);
|
|
|
|
var today = RangeParts.LocalDate(Now, _zone);
|
|
var manual = await _db.ManualCosts.AsNoTracking()
|
|
.Where(c => c.PeriodStart <= today)
|
|
.Select(c => new { c.MeterId, c.PeriodStart })
|
|
.ToListAsync(cancellationToken).ConfigureAwait(false);
|
|
var scopeMeters = ScopeMeters().ToHashSet();
|
|
var days = manual
|
|
.Where(c => _request.Scope.Kind == AnalysisScopeKind.Portfolio || (c.MeterId is { } id && scopeMeters.Contains(id)))
|
|
.Select(c => c.PeriodStart)
|
|
.ToList();
|
|
|
|
AvailableRange? manualRange = null;
|
|
if (days.Count > 0)
|
|
{
|
|
var from = Core.Normalization.GapAttribution.LocalMidnight(days.Min(), _zone);
|
|
var to = Core.Normalization.GapAttribution.LocalMidnight(days.Max().AddDays(1), _zone);
|
|
manualRange = AvailableRange.Of(from, to < Now ? to : Now, _zone) ?? AvailableRange.Of(from, to, _zone);
|
|
}
|
|
|
|
var cost = AvailableRange.Union([metered, manualRange], _zone);
|
|
LatestPeriod? latest = null;
|
|
if (cost is not null)
|
|
{
|
|
var month = cost.LatestMonth;
|
|
var byMeters = metered?.LatestMonth == month;
|
|
var byManual = manualRange?.LatestMonth == month;
|
|
latest = new LatestPeriod(month, byMeters && byManual ? LatestPeriodBasis.Both : byManual ? LatestPeriodBasis.Manual : LatestPeriodBasis.Meters);
|
|
}
|
|
|
|
return new ScopeAvailability(quantity, cost, latest);
|
|
}
|
|
|
|
/// <summary>The attention items (D-53) of everything the result shows.</summary>
|
|
private void CollectProblems()
|
|
{
|
|
var shown = _seriesIds.Concat(_measures.SelectMany(m => m.Group.MeterIds)).Distinct().ToList();
|
|
var virtuals = new SortedSet<int>();
|
|
foreach (var id in shown)
|
|
{
|
|
CollectVirtuals(id, virtuals);
|
|
}
|
|
|
|
foreach (var id in virtuals)
|
|
{
|
|
var meter = _catalog.Meters[id];
|
|
switch (meter.VirtualStatus)
|
|
{
|
|
case VirtualMeterStatus.Legacy:
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.LegacyDefinition, id)
|
|
{
|
|
MeterIds = meter.Legacy?.MeterIds ?? [],
|
|
Legacy = meter.Legacy?.Outcome,
|
|
});
|
|
break;
|
|
case VirtualMeterStatus.NeedsConfiguration:
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.LegacyNeedsConfiguration, id)
|
|
{
|
|
MeterIds = meter.Legacy?.MeterIds ?? [],
|
|
Values = meter.Legacy?.Values ?? [],
|
|
Legacy = meter.Legacy?.Outcome,
|
|
});
|
|
break;
|
|
case VirtualMeterStatus.Malformed:
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.MalformedDefinition, id)
|
|
{
|
|
Values = meter.StoredDefinition?.Problem is { } malformation ? [malformation] : [],
|
|
});
|
|
break;
|
|
case VirtualMeterStatus.Invalid:
|
|
foreach (var problem in meter.Validation?.Problems ?? [])
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.InvalidDefinition, id)
|
|
{
|
|
MeterIds = problem.MeterIds,
|
|
Values = problem.Values,
|
|
Virtual = problem,
|
|
});
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
foreach (var id in _dataLeaves.Order())
|
|
{
|
|
if (_leaves[id].Meter.IsPending)
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.AnalysisPending, id));
|
|
}
|
|
|
|
if (_afterNow.GetValueOrDefault(id) is { } block)
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.RecordedAfterNow, id) { AfterNow = block });
|
|
}
|
|
|
|
if (_freshness.GetValueOrDefault(id) is { State: FreshnessState.Stale })
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.StaleSource, id));
|
|
}
|
|
}
|
|
|
|
if (_request.Scope.Kind == AnalysisScopeKind.Meters)
|
|
{
|
|
return;
|
|
}
|
|
|
|
var types = TypesInScope();
|
|
foreach (var problem in _catalog.Totals.Problems.Where(p => _catalog.Find(p.MeterId) is { } m && types.Contains(m.EnergyTypeId)))
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.TotalsProblem, problem.MeterId)
|
|
{
|
|
MeterIds = problem.OtherMeterId is { } other ? [other] : [],
|
|
Totals = problem,
|
|
});
|
|
}
|
|
|
|
foreach (var hint in _catalog.Totals.Hints.Where(h => types.Contains(h.EnergyTypeId)))
|
|
{
|
|
_problems.Add(new AnalysisProblem(AnalysisProblemKind.PossibleOverlap, hint.MeterId) { MeterIds = [hint.OtherMeterId], Hint = hint });
|
|
}
|
|
}
|
|
|
|
private void CollectVirtuals(int meterId, SortedSet<int> found)
|
|
{
|
|
if (_catalog.Find(meterId) is not { IsVirtual: true } meter || !found.Add(meterId))
|
|
{
|
|
return;
|
|
}
|
|
|
|
foreach (var source in meter.Formula?.MeterIds ?? [])
|
|
{
|
|
CollectVirtuals(source, found);
|
|
}
|
|
}
|
|
|
|
private HashSet<int> TypesInScope() => _request.Scope.Kind == AnalysisScopeKind.EnergyType
|
|
? [_request.Scope.EnergyTypeId!.Value]
|
|
: [.. _catalog.Meters.Values.Select(m => m.EnergyTypeId)];
|
|
|
|
private IReadOnlyList<MeterClassification> Classification()
|
|
{
|
|
if (_request.Scope.Kind == AnalysisScopeKind.Meters)
|
|
{
|
|
return [];
|
|
}
|
|
|
|
return [.. ScopeMeters().Order()
|
|
.Where(id => _catalog.Totals.Meters.ContainsKey(id))
|
|
.Select(id => new MeterClassification(id, _catalog.Meters[id].Name, _catalog.Totals.Meters[id]))];
|
|
}
|
|
|
|
private static List<AnalysisProblem> Deduplicated(List<AnalysisProblem> problems)
|
|
{
|
|
var seen = new HashSet<(AnalysisProblemKind, int?, VirtualProblemKind?, TotalsProblemKind?, OverlapHintKind?)>();
|
|
return [.. problems.Where(p => seen.Add((p.Kind, p.MeterId, p.Virtual?.Kind, p.Totals?.Kind, p.Hint?.Kind)))];
|
|
}
|
|
|
|
private static string MeterKey(int meterId) => SeriesKey.ForMeter(meterId, 0, string.Empty).Id;
|
|
|
|
private static string MeasureKey(int energyTypeId, MeasureGroup group) => SeriesKey.ForMeasure(energyTypeId, group.Measure, group.Unit).Id;
|
|
|
|
/// <summary>The values of one physical meter on one side.</summary>
|
|
private sealed record PhysicalValues(
|
|
IReadOnlyList<BucketValue> Values, BucketValue Total, IReadOnlyList<BucketCoverage>? Coverage, BucketCoverage? TotalCoverage);
|
|
|
|
/// <summary>
|
|
/// One side of a request — the current period or its comparison — with its buckets, the bucket that is the whole
|
|
/// period, the cut-off its actuals stop at, and what has been computed for it.
|
|
/// </summary>
|
|
private sealed class Side(IReadOnlyList<AnalysisBucket> buckets, AnalysisBucket periodBucket, DateTimeOffset cutoff)
|
|
{
|
|
private List<DayWindow>? _dayWindows;
|
|
|
|
public IReadOnlyList<AnalysisBucket> Buckets { get; } = buckets;
|
|
|
|
public AnalysisBucket PeriodBucket { get; } = periodBucket;
|
|
|
|
public DateTimeOffset Cutoff { get; } = cutoff;
|
|
|
|
public Dictionary<int, (Tally[] Buckets, Tally Total)> Tallies { get; } = [];
|
|
|
|
public Dictionary<(int, bool), PhysicalValues> Physical { get; } = [];
|
|
|
|
public Dictionary<int, VirtualEvaluation?> Evaluations { get; } = [];
|
|
|
|
public Dictionary<int, VirtualSource> Sources { get; } = [];
|
|
|
|
/// <summary>The local days the buckets span, each clipped to the buckets' range.</summary>
|
|
public List<DayWindow> DayWindows(TimeZoneInfo zone)
|
|
{
|
|
if (_dayWindows is not null)
|
|
{
|
|
return _dayWindows;
|
|
}
|
|
|
|
_dayWindows = [];
|
|
if (Buckets.Count == 0)
|
|
{
|
|
return _dayWindows;
|
|
}
|
|
|
|
var from = Buckets[0].From;
|
|
var to = Buckets[^1].To;
|
|
if (to <= from)
|
|
{
|
|
return _dayWindows;
|
|
}
|
|
|
|
for (var day = RangeParts.LocalDate(from, zone); ; day = day.AddDays(1))
|
|
{
|
|
var start = Core.Normalization.GapAttribution.LocalMidnight(day, zone);
|
|
if (start >= to)
|
|
{
|
|
break;
|
|
}
|
|
|
|
var end = Core.Normalization.GapAttribution.LocalMidnight(day.AddDays(1), zone);
|
|
var windowFrom = start > from ? start : from;
|
|
var windowTo = end < to ? end : to;
|
|
if (windowTo > windowFrom)
|
|
{
|
|
_dayWindows.Add(new DayWindow(day, windowFrom, windowTo));
|
|
}
|
|
}
|
|
|
|
return _dayWindows;
|
|
}
|
|
}
|
|
|
|
/// <summary>One local day of a side, clipped to its range.</summary>
|
|
private readonly record struct DayWindow(DateOnly Day, DateTimeOffset From, DateTimeOffset To);
|
|
}
|