From a7d01212055f43d5865fc714f81575f3b05aa7dc Mon Sep 17 00:00:00 2001 From: Bill Bai Date: Wed, 3 Jun 2026 21:19:33 -0500 Subject: [PATCH 1/4] feat(shifts): named/wrapping shift model (Shift/ShiftSet) + aggregate/occurrence/comparison Add the core generalized shift-splitting model to analysis.py: - Shift / ShiftSet with parse(), from_spec(), default(), HH:MM validation, duplicate-name + zero-length-window rejection, and wrap-past-midnight windows (end <= start). - ShiftSet.coverage_issues() warns on gaps/overlaps in the 24h tiling. - gather_store(store, indices): slice_store sibling over non-contiguous index lists (a named shift recurs daily). - aggregate_shifts(): group record indices by shift NAME, evaluated in the REPORT timezone (localize before applying HH:MM rule). First matching window wins; non-matching records -> 'unassigned'. - shift_occurrences(): contiguous per-instance buckets; a midnight-spanning night is ONE occurrence labeled by its start date. - shift_comparison_rows() + _shift_row(): the headline per-shift aggregate (records, hours, kWh, P avg/min/max, peak rolling demand, PF avg, V_LN avg/p5/p95, V_THD p95, event counts, outage minutes). 20 new tests cover parsing, wrap logic, tz-localized assignment, non-contiguous aggregation, gap/overlap/unassigned, occurrences (incl. midnight span), and comparison-row schema/values. Full suite: 243 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/src/fluke_3540/analysis.py | 339 ++++++++++++++++++++++++++++++ python/tests/test_shifts.py | 318 ++++++++++++++++++++++++++++ 2 files changed, 657 insertions(+) create mode 100644 python/tests/test_shifts.py diff --git a/python/src/fluke_3540/analysis.py b/python/src/fluke_3540/analysis.py index d99de0a..8b01ee4 100644 --- a/python/src/fluke_3540/analysis.py +++ b/python/src/fluke_3540/analysis.py @@ -610,6 +610,345 @@ def slice_store(store: ColumnStore, lo: int, hi: int) -> ColumnStore: return sub +def gather_store(store: ColumnStore, indices: Sequence[int]) -> ColumnStore: + """Like :func:`slice_store` but over an arbitrary (possibly non-contiguous) + list of record indices. + + Used by shift aggregation: a "night" shift recurs daily, so its records + are scattered through the store rather than forming one contiguous range. + ``indices`` are taken in the given order (callers pass them ascending so + timestamps stay monotonic, which keeps kWh/demand roll-ups well-defined). + """ + sub = ColumnStore(time_shift=store.time_shift) + cols = store.columns + src = {name: store._cols[name] for name in cols} + dst = {name: sub._cols[name] for name in cols} + s_ticks = store._start_ticks + e_ticks = store._end_ticks + for i in indices: + for name in cols: + dst[name].append(src[name][i]) + sub._start_ticks.append(s_ticks[i]) + sub._end_ticks.append(e_ticks[i]) + sub._n = len(sub._start_ticks) + return sub + + +# --- Generalized named shift/period splitting (--split-by shifts) ------------ + +UNASSIGNED_SHIFT = "unassigned" + + +def _parse_hhmm(text: str) -> int: + """Parse 'HH:MM' (colon required) to minute-of-day in [0, 1440]. + + '24:00' is accepted as end-of-day (1440). Raises ValueError otherwise. + """ + s = text.strip() + if ":" not in s: + raise ValueError(f"shift time must be HH:MM (with a colon): {text!r}") + hh, _, mm = s.partition(":") + if not (hh.isdigit() and mm.isdigit()): + raise ValueError(f"shift time must be numeric HH:MM: {text!r}") + h = int(hh) + m = int(mm) + if m >= 60: + raise ValueError(f"shift minutes out of range in {text!r}") + total = h * 60 + m + if total < 0 or total > 1440: + raise ValueError(f"shift time out of range (00:00..24:00): {text!r}") + return total + + +@dataclass(frozen=True) +class Shift: + """One named shift window, in minutes-of-day (report-tz wall clock). + + ``end_min <= start_min`` means the window wraps past midnight, e.g. + 18:00-06:00 covers [18:00, 24:00) plus [00:00, 06:00). + """ + name: str + start_min: int + end_min: int + + @property + def wraps(self) -> bool: + return self.end_min <= self.start_min + + def contains_minute(self, mod: int) -> bool: + """Is minute-of-day ``mod`` inside this window? start inclusive, end + exclusive; wrap-aware.""" + if not self.wraps: + return self.start_min <= mod < self.end_min + # Wrapping: [start, 1440) U [0, end) + return mod >= self.start_min or mod < self.end_min + + def length_minutes(self) -> int: + if self.wraps: + return (1440 - self.start_min) + self.end_min + return self.end_min - self.start_min + + @property + def window_str(self) -> str: + def fmt(m: int) -> str: + return f"{(m // 60) % 24:02d}:{m % 60:02d}" if m != 1440 else "24:00" + return f"{fmt(self.start_min)}-{fmt(self.end_min)}" + + +@dataclass(frozen=True) +class ShiftSet: + """An ordered collection of named shift windows. + + Order matters: each record is assigned to the FIRST window it matches, so + overlaps resolve deterministically. + """ + shifts: tuple[Shift, ...] + + @classmethod + def parse(cls, text: str) -> "ShiftSet": + """Parse 'name=HH:MM-HH:MM,name=HH:MM-HH:MM,...'.""" + out: list[Shift] = [] + seen: set[str] = set() + parts = [p.strip() for p in text.split(",") if p.strip()] + if not parts: + raise ValueError(f"no shifts parsed from {text!r}") + for part in parts: + name, sep, window = part.partition("=") + name = name.strip() + if not sep or not name: + raise ValueError( + f"shift must be name=HH:MM-HH:MM: {part!r}") + a, dash, b = window.partition("-") + if not dash: + raise ValueError( + f"shift window must be HH:MM-HH:MM: {window!r}") + start = _parse_hhmm(a) + end = _parse_hhmm(b) + if start == end: + raise ValueError( + f"shift {name!r} has a zero-length window {window!r}") + if name in seen: + raise ValueError(f"duplicate shift name {name!r}") + seen.add(name) + out.append(Shift(name, start, end)) + return cls(tuple(out)) + + @classmethod + def from_spec(cls, spec: Sequence[dict]) -> "ShiftSet": + """Build from a list of {name, start, end} dicts (the --shifts-file form).""" + text = ",".join( + f"{s['name']}={s['start']}-{s['end']}" for s in spec) + return cls.parse(text) + + @classmethod + def default(cls) -> "ShiftSet": + return cls.parse("day=06:00-18:00,night=18:00-06:00") + + # --- validation -------------------------------------------------------- + def coverage_issues(self) -> list[str]: + """Warn if the windows don't tile 24 h (gaps or overlaps). + + Walks minute-of-day 0..1439 counting how many windows cover each + minute. Reports the count of uncovered minutes (gap) and doubly-covered + minutes (overlap). Empty list = clean 24 h tiling. + """ + cover = [0] * 1440 + for sh in self.shifts: + for m in range(1440): + if sh.contains_minute(m): + cover[m] += 1 + gap = sum(1 for c in cover if c == 0) + overlap = sum(1 for c in cover if c > 1) + issues: list[str] = [] + if gap: + issues.append( + f"{gap} minute(s)/day fall in NO shift window (gap); those " + f"records go to '{UNASSIGNED_SHIFT}'.") + if overlap: + issues.append( + f"{overlap} minute(s)/day are covered by MORE THAN ONE shift " + "(overlap); the first matching window wins.") + return issues + + +def _localize_minute(t: dt.datetime, tz) -> int: + """Minute-of-day for ``t`` in the report timezone ``tz`` (UTC if None).""" + if tz is not None: + t = t.astimezone(tz) + return t.hour * 60 + t.minute + + +def aggregate_shifts(store: ColumnStore, ss: ShiftSet, tz=None) -> dict[str, list[int]]: + """Group record indices by shift NAME, evaluating windows in ``tz``. + + Returns {shift_name: [ascending indices]}. Each named shift present in + ``ss`` always has a key (possibly empty). Records matching no window are + collected under ``UNASSIGNED_SHIFT`` (only added when non-empty). + """ + out: dict[str, list[int]] = {sh.name: [] for sh in ss.shifts} + for i in range(store.n): + mod = _localize_minute(store.start(i), tz) + for sh in ss.shifts: + if sh.contains_minute(mod): + out[sh.name].append(i) + break + else: + out.setdefault(UNASSIGNED_SHIFT, []).append(i) + return out + + +def shift_occurrences(store: ColumnStore, ss: ShiftSet, tz=None + ) -> list[tuple[str, str, int, int]]: + """Partition records into contiguous per-OCCURRENCE shift buckets. + + Returns a time-ordered list of (label, shift_name, lo, hi_exclusive). A + new occurrence starts whenever the matched shift name changes between + consecutive records. A wrap-past-midnight window stays ONE occurrence + (the name does not change), labeled by the occurrence's START date in the + report timezone, e.g. ``"night 2026-05-29"``. + """ + n = store.n + if n == 0: + return [] + + def name_at(i: int) -> str: + mod = _localize_minute(store.start(i), tz) + for sh in ss.shifts: + if sh.contains_minute(mod): + return sh.name + return UNASSIGNED_SHIFT + + def start_date(i: int) -> str: + t = store.start(i) + if tz is not None: + t = t.astimezone(tz) + return t.strftime("%Y-%m-%d") + + out: list[tuple[str, str, int, int]] = [] + cur_name = name_at(0) + lo = 0 + for i in range(1, n): + nm = name_at(i) + if nm != cur_name: + out.append((f"{cur_name} {start_date(lo)}", cur_name, lo, i)) + cur_name = nm + lo = i + out.append((f"{cur_name} {start_date(lo)}", cur_name, lo, n)) + return out + + +def shift_comparison_rows(store: ColumnStore, ss: ShiftSet, events: Sequence, + tz=None, nominal_ln_v: float = 277.0, + demand_window: int = 15) -> list[dict]: + """The headline per-shift-name aggregate comparison. + + One row per named shift (plus ``unassigned`` if any records land there), + aggregating ALL records of that shift across the whole session. ``events`` + are filed to a shift by the shift their ``t_start`` falls in. + + ``demand_window`` is in MINUTES (peak rolling demand within the shift's + records). Per-row energy/avg/percentiles are memory-bounded. + """ + by_name = aggregate_shifts(store, ss, tz=tz) + win_by_name = {sh.name: sh for sh in ss.shifts} + + # File events to a shift by their start time's window. + ev_by_name: dict[str, list] = {nm: [] for nm in by_name} + for e in events: + mod = _localize_minute(e.t_start, tz) + placed = False + for sh in ss.shifts: + if sh.contains_minute(mod): + ev_by_name.setdefault(sh.name, []).append(e) + placed = True + break + if not placed: + ev_by_name.setdefault(UNASSIGNED_SHIFT, []).append(e) + + rows: list[dict] = [] + for name, idxs in by_name.items(): + sub = gather_store(store, idxs) + sh = win_by_name.get(name) + window = sh.window_str if sh is not None else "—" + bucket_events = ev_by_name.get(name, []) + rows.append(_shift_row(name, window, sub, bucket_events, + nominal_ln_v, demand_window)) + return rows + + +def _shift_row(name: str, window: str, sub: ColumnStore, bucket_events, + nominal_ln_v: float, demand_window: int) -> dict: + """Aggregate one shift's records into a comparison row.""" + nrec = sub.n + p = sub.col("P_total_avg_W") + pf = sub.col("PF_total_avg") + va = sub.col("V_LN_a_avg_V"); vb = sub.col("V_LN_b_avg_V"); vc = sub.col("V_LN_c_avg_V") + vth_a = sub.col("V_THD_pct_a_avg") + vth_b = sub.col("V_THD_pct_b_avg") + vth_c = sub.col("V_THD_pct_c_avg") + + p_mom = _RunningMoments() + p_min = math.inf; p_max = -math.inf + pf_mom = _RunningMoments() + v_mom = _RunningMoments() + v_sketch = _PercentileSketch(0.0, 400.0) + vthd_sketch = _PercentileSketch(0.0, 50.0) + for i in range(nrec): + pv = p[i] + if math.isfinite(pv): + p_mom.add(pv) + p_min = min(p_min, pv); p_max = max(p_max, pv) + pfi = pf[i] + if math.isfinite(pfi): + pf_mom.add(pfi) + for vv in (va[i], vb[i], vc[i]): + if math.isfinite(vv) and vv > 50.0: # ignore outage zeros + v_mom.add(vv); v_sketch.add(vv) + for tv in (vth_a[i], vth_b[i], vth_c[i]): + if math.isfinite(tv): + vthd_sketch.add(tv) + + p_mean = p_mom.mean if p_mom.n else 0.0 + hours = p_mom.n / 3600.0 # 1 record == 1 s + kwh = p_mean / 1000.0 * hours + + # Peak rolling demand within the shift's gathered records. + demand = demand_analysis(sub, window_secs=max(1, demand_window * 60)) + + n_out = sum(1 for e in bucket_events if e.kind == "outage") + n_dip = sum(1 for e in bucket_events if e.kind == "dip") + n_swell = sum(1 for e in bucket_events if e.kind == "swell") + outage_minutes = sum( + (e.t_end - e.t_start).total_seconds() for e in bucket_events + if e.kind == "outage") / 60.0 + + def q(sketch, p_): + v = sketch.quantile(p_) + return 0.0 if (v != v) else v + + return { + "shift": name, + "window": window, + "records": nrec, + "hours": hours, + "kWh": kwh, + "P_total_avg_W": p_mean, + "P_total_min_W": (p_min if p_min != math.inf else 0.0), + "P_total_max_W": (p_max if p_max != -math.inf else 0.0), + "peak_demand_kW": demand["peak_demand_kw"], + "peak_demand_window_secs": demand["window_secs"], + "PF_avg": (pf_mom.mean if pf_mom.n else 0.0), + "V_LN_avg_V": (v_mom.mean if v_mom.n else 0.0), + "V_LN_p5_V": q(v_sketch, 0.05), + "V_LN_p95_V": q(v_sketch, 0.95), + "V_THD_p95_pct": q(vthd_sketch, 0.95), + "n_outages": n_out, + "n_dips": n_dip, + "n_swells": n_swell, + "outage_minutes": outage_minutes, + } + + # --- Event markers / correlation (--mark / --marks) ------------------------- @dataclass(frozen=True) diff --git a/python/tests/test_shifts.py b/python/tests/test_shifts.py new file mode 100644 index 0000000..24dd008 --- /dev/null +++ b/python/tests/test_shifts.py @@ -0,0 +1,318 @@ +"""Tests for generalized, named, configurable shift/period splitting. + +The shift model lets users define multiple named time windows within a day +(which may wrap past midnight) and: + * AGGREGATE all records of each named shift across the whole session into a + single comparison row (day vs night, A/B/C shifts, …); + * bucket each individual shift OCCURRENCE as its own contiguous time-ordered + range (a "night" spanning midnight is ONE occurrence labeled by its start + date). + +CRITICAL tz contract: the store holds UTC timestamps; shift windows are +evaluated in the *report* timezone (``--tz``). The localize step happens before +the HH:MM window rule is applied. +""" +from __future__ import annotations + +import datetime as dt + +import pytest + +try: # Python 3.9+ + from zoneinfo import ZoneInfo +except ImportError: # pragma: no cover + ZoneInfo = None + +from fluke_3540.analysis import ( + Shift, + ShiftSet, + aggregate_shifts, + gather_store, + shift_comparison_rows, + shift_occurrences, +) +from fluke_3540.parser import Record +from fluke_3540.store import ColumnStore + +from conftest import make_records + + +def make_minute_records(count, base, overrides=None, defaults=None): + """make_records spaced one MINUTE apart (the per-second helper packs all + records into the same minute-of-day, which the shift logic keys on).""" + recs = make_records(count, base=base, overrides=overrides, defaults=defaults) + out = [] + for n, r in enumerate(recs): + start = base + dt.timedelta(minutes=n) + out.append(Record(index=n, start=start, + end=start + dt.timedelta(minutes=1), floats=r.floats)) + return out + + +# --- ShiftSet.parse ---------------------------------------------------------- + +def test_parse_two_shifts(): + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + assert [s.name for s in ss.shifts] == ["day", "night"] + day, night = ss.shifts + assert (day.start_min, day.end_min) == (360, 1080) + assert day.wraps is False + assert (night.start_min, night.end_min) == (1080, 360) + assert night.wraps is True + + +def test_parse_three_shifts(): + ss = ShiftSet.parse("A=06:00-14:00,B=14:00-22:00,C=22:00-06:00") + assert [s.name for s in ss.shifts] == ["A", "B", "C"] + assert ss.shifts[2].wraps is True # C crosses midnight + + +def test_parse_default(): + ss = ShiftSet.default() + assert [s.name for s in ss.shifts] == ["day", "night"] + assert ss.shifts[0].start_min == 360 + + +def test_parse_bad_time(): + with pytest.raises(ValueError): + ShiftSet.parse("day=06:00-25:00") + with pytest.raises(ValueError): + ShiftSet.parse("day=0600-1800") # missing colon → not HH:MM + with pytest.raises(ValueError): + ShiftSet.parse("garbage") + + +def test_parse_duplicate_names(): + with pytest.raises(ValueError): + ShiftSet.parse("day=06:00-12:00,day=12:00-18:00") + + +def test_parse_zero_length_window_rejected(): + with pytest.raises(ValueError): + ShiftSet.parse("x=06:00-06:00") + + +def test_from_spec_dicts(): + ss = ShiftSet.from_spec([ + {"name": "day", "start": "06:00", "end": "18:00"}, + {"name": "night", "start": "18:00", "end": "06:00"}, + ]) + assert [s.name for s in ss.shifts] == ["day", "night"] + assert ss.shifts[1].wraps is True + + +# --- Shift.contains_minute (wrap logic) -------------------------------------- + +def test_contains_minute_non_wrapping(): + day = Shift("day", 360, 1080) + assert day.contains_minute(360) is True # 06:00 inclusive + assert day.contains_minute(1079) is True # 17:59 + assert day.contains_minute(1080) is False # 18:00 exclusive + assert day.contains_minute(300) is False + + +def test_contains_minute_wrapping(): + night = Shift("night", 1080, 360) + assert night.contains_minute(1080) is True # 18:00 + assert night.contains_minute(1439) is True # 23:59 + assert night.contains_minute(0) is True # 00:00 + assert night.contains_minute(359) is True # 05:59 + assert night.contains_minute(360) is False # 06:00 → day + assert night.contains_minute(720) is False # noon + + +# --- tz-localized assignment ------------------------------------------------- + +@pytest.mark.skipif(ZoneInfo is None, reason="zoneinfo unavailable") +def test_assignment_localizes_to_report_tz(): + # A record stored at 10:30 UTC is 05:30 America/Chicago (CDT, UTC-5). + # In Central that lands in the night shift (the 00:00-06:00 wrap leg); in + # UTC (10:30, mod 630) it would land in the day shift. This pins the tz + # contract: windows are evaluated in the report tz, not raw UTC. + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + central = ZoneInfo("America/Chicago") + base = dt.datetime(2026, 5, 29, 10, 30, 0, tzinfo=dt.timezone.utc) + recs = make_records(1, base=base) + store = ColumnStore.from_records(recs) + + by_name_central = aggregate_shifts(store, ss, tz=central) + assert by_name_central["night"] == [0] + assert by_name_central.get("day", []) == [] + + by_name_utc = aggregate_shifts(store, ss, tz=None) + assert by_name_utc["day"] == [0] + assert by_name_utc.get("night", []) == [] + + +def test_assignment_utc_default(): + # 08:00 UTC, no tz → day shift. + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + base = dt.datetime(2026, 5, 29, 8, 0, 0, tzinfo=dt.timezone.utc) + recs = make_records(1, base=base) + store = ColumnStore.from_records(recs) + by_name = aggregate_shifts(store, ss, tz=None) + assert by_name["day"] == [0] + + +# --- aggregate_shifts grouping (non-contiguous) ------------------------------ + +def test_aggregate_groups_noncontiguous_indices(): + # 06:00 + a few hours of records straddling into night, all 1/sec. Build a + # compact multi-window day: 11 records starting 17:58:00 UTC → 17:58..18:08. + # Records before 18:00 → day; 18:00 onward → night. So indices are split + # into two contiguous chunks here, but the API returns lists by name. + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + base = dt.datetime(2026, 5, 29, 17, 58, 0, tzinfo=dt.timezone.utc) + recs = make_minute_records(11, base=base) # 17:58 .. 18:08 starts + store = ColumnStore.from_records(recs) + by_name = aggregate_shifts(store, ss, tz=None) + # 17:58:00, 17:59:00 (mins 1078,1079) are day; 18:00:00.. are night. + assert by_name["day"] == [0, 1] + assert by_name["night"] == list(range(2, 11)) + + +def test_gather_store_picks_indices(): + recs = make_records(50, defaults={"P_total_avg_W": 1000.0}) + store = ColumnStore.from_records(recs, time_shift=dt.timedelta(hours=2)) + sub = gather_store(store, [3, 7, 40]) + assert sub.n == 3 + assert sub.time_shift == dt.timedelta(hours=2) + assert sub.start(0) == store.start(3) + assert sub.start(1) == store.start(7) + assert sub.col("P_total_avg_W")[2] == store.col("P_total_avg_W")[40] + + +# --- unassigned + overlap/gap validation ------------------------------------- + +def test_unassigned_when_window_does_not_tile(): + # Only a morning shift; afternoon records have no home → "unassigned". + ss = ShiftSet.parse("morning=06:00-12:00") + base = dt.datetime(2026, 5, 29, 11, 58, 0, tzinfo=dt.timezone.utc) + recs = make_minute_records(5, base=base) # 11:58..12:02 + store = ColumnStore.from_records(recs) + by_name = aggregate_shifts(store, ss, tz=None) + assert by_name["morning"] == [0, 1] # 11:58, 11:59 + assert by_name["unassigned"] == [2, 3, 4] # 12:00, 12:01, 12:02 + + +def test_first_matching_window_wins_on_overlap(): + # Overlapping windows: record at 10:00 matches both; the first listed wins. + ss = ShiftSet.parse("early=06:00-12:00,late=10:00-18:00") + base = dt.datetime(2026, 5, 29, 10, 0, 0, tzinfo=dt.timezone.utc) + recs = make_records(1, base=base) + store = ColumnStore.from_records(recs) + by_name = aggregate_shifts(store, ss, tz=None) + assert by_name["early"] == [0] + assert by_name.get("late", []) == [] + + +def test_coverage_gaps_and_overlaps_reported(): + full = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + assert full.coverage_issues() == [] # tiles 24 h cleanly + + gapped = ShiftSet.parse("morning=06:00-12:00,evening=14:00-20:00") + issues = gapped.coverage_issues() + assert any("gap" in s.lower() for s in issues) + + overlapped = ShiftSet.parse("early=06:00-13:00,late=12:00-18:00") + issues = overlapped.coverage_issues() + assert any("overlap" in s.lower() for s in issues) + + +# --- occurrences (contiguous per-instance buckets) --------------------------- + +def test_shift_occurrences_labels_by_start_date(): + # Night window 18:00-06:00; records 17:58:00..18:03:00 UTC → the records at + # 18:00+ are one "night" occurrence labeled by its start date. + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + base = dt.datetime(2026, 5, 29, 17, 58, 0, tzinfo=dt.timezone.utc) + recs = make_minute_records(6, base=base) # 17:58..18:03 starts + store = ColumnStore.from_records(recs) + occ = shift_occurrences(store, ss, tz=None) + # occurrences are (label, name, lo, hi) contiguous ranges, time-ordered. + labels = [o[0] for o in occ] + names = [o[1] for o in occ] + assert names == ["day", "night"] + assert labels[1] == "night 2026-05-29" + # contiguous & covering + assert occ[0][2] == 0 and occ[-1][3] == store.n + for a, b in zip(occ, occ[1:]): + assert a[3] == b[2] + + +def test_shift_occurrences_midnight_span_is_one_bucket(): + # Build records crossing local midnight inside the night shift. Use UTC tz + # so wall = stored. 23:58:00 .. 00:02:00 (next day) → all night, ONE + # occurrence labeled by the START date (the 29th). + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + base = dt.datetime(2026, 5, 29, 23, 58, 0, tzinfo=dt.timezone.utc) + recs = make_minute_records(5, base=base) # 23:58,23:59,00:00,00:01,00:02 + store = ColumnStore.from_records(recs) + occ = shift_occurrences(store, ss, tz=None) + night_occ = [o for o in occ if o[1] == "night"] + assert len(night_occ) == 1 + assert night_occ[0][0] == "night 2026-05-29" + assert night_occ[0][2] == 0 and night_occ[0][3] == 5 + + +# --- comparison rows (the headline output) ----------------------------------- + +def test_shift_comparison_rows_schema_and_values(): + # Two shifts, distinct power levels so the comparison is meaningful. + # Build 4 minutes: 2 min in day (05:59 → no, use 06:00..) and 2 in night. + # Easier: 17:59:00 (day) ×60 then 18:00:00 (night) ×60, each 1/sec. + overrides = {} + for i in range(60): + overrides[i] = {"P_total_avg_W": 10_000.0} # day records + for i in range(60, 120): + overrides[i] = {"P_total_avg_W": 20_000.0} # night records + base = dt.datetime(2026, 5, 29, 17, 59, 0, tzinfo=dt.timezone.utc) + recs = make_records(120, base=base, overrides=overrides) + store = ColumnStore.from_records(recs) + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + rows = shift_comparison_rows(store, ss, events=[], tz=None, + nominal_ln_v=277.0, demand_window=15) + by = {r["shift"]: r for r in rows} + assert set(by) == {"day", "night"} + d, ngt = by["day"], by["night"] + assert d["records"] == 60 + assert ngt["records"] == 60 + assert d["window"] == "06:00-18:00" + assert ngt["window"] == "18:00-06:00" + assert d["P_total_avg_W"] == pytest.approx(10_000.0) + assert ngt["P_total_avg_W"] == pytest.approx(20_000.0) + assert ngt["P_total_avg_W"] > d["P_total_avg_W"] + # energy = mean power * hours; 60 records = 60 s = 1/60 h. + assert d["kWh"] == pytest.approx(10.0 / 1000.0 * (60 / 3600.0) * 1000) # 10 kW * (60s/3600) h + # required schema keys present + for k in ("shift", "window", "records", "hours", "kWh", "P_total_avg_W", + "P_total_min_W", "P_total_max_W", "peak_demand_kW", "PF_avg", + "V_LN_avg_V", "V_LN_p5_V", "V_LN_p95_V", "V_THD_p95_pct", + "n_outages", "n_dips", "n_swells", "outage_minutes"): + assert k in d, f"missing schema key {k}" + + +def test_shift_comparison_multi_day_aggregates_across_occurrences(): + # Two day-shift windows on two different dates aggregate into ONE day row. + # Day A: 2026-05-29 08:00 ×30 @ 5kW ; Day B: 2026-05-30 08:00 ×30 @ 15kW. + # Using a 1-day gap of night records between them so they're non-contiguous. + recs = [] + recs += make_records(30, base=dt.datetime(2026, 5, 29, 8, 0, tzinfo=dt.timezone.utc), + defaults={"P_total_avg_W": 5_000.0}) + recs += make_records(30, base=dt.datetime(2026, 5, 29, 20, 0, tzinfo=dt.timezone.utc), + defaults={"P_total_avg_W": 1_000.0}) # night + recs += make_records(30, base=dt.datetime(2026, 5, 30, 8, 0, tzinfo=dt.timezone.utc), + defaults={"P_total_avg_W": 15_000.0}) + # Re-index the records so indices are contiguous 0..89 with correct times. + from fluke_3540.parser import Record + fixed = [Record(index=i, start=r.start, end=r.end, floats=r.floats) + for i, r in enumerate(recs)] + store = ColumnStore.from_records(fixed) + ss = ShiftSet.parse("day=06:00-18:00,night=18:00-06:00") + rows = shift_comparison_rows(store, ss, events=[], tz=None, + nominal_ln_v=277.0, demand_window=15) + by = {r["shift"]: r for r in rows} + assert by["day"]["records"] == 60 # both day windows merged + assert by["night"]["records"] == 30 + # day avg power = mean of 30×5k + 30×15k = 10k + assert by["day"]["P_total_avg_W"] == pytest.approx(10_000.0) From 9bb205da3453096abb284f2a7573358d4bfeca6d Mon Sep 17 00:00:00 2001 From: Bill Bai Date: Wed, 3 Jun 2026 21:27:29 -0500 Subject: [PATCH 2/4] feat(shifts): wire --split-by shifts + --shifts/--shifts-file into CLI - New shifts_file.py loader (JSON {shifts:[{name,start,end}]} or bare list), mirroring the --rules-file pattern. - CLI: --split-by shifts routes to _run_shifts; --shifts SPEC (inline) and --shifts-file FILE; default day=06:00-18:00,night=18:00-06:00 when neither given. Windows evaluated in --tz (UTC if unset; printed in output). - _run_shifts emits the two required outputs: 1. shift_comparison.csv + .json (headline per-shift aggregate; tz/spec/coverage_issues/demand_window in the JSON header), 2. per-occurrence contiguous buckets under /shifts// (session.csv, events.json+ITIC, summary.txt), reusing per-bucket machinery; midnight-spanning occurrences are one bucket by start date. - Coverage gap/overlap warnings printed to stderr. - Shift-comparison table embedded in summary.txt. Validated read-only on the real ES.004 session with --tz America/Chicago: day/night split lands exactly on 06:00/18:00 Central (43200 rec each per day). Tests: 6 CLI E2E (default, boundary-cross, tz-central, shifts-file, 3-shift, summary-table) + 3 shifts_file loader. Full suite: 251 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/src/fluke_3540/cli.py | 154 +++++++++++++++++++++++++-- python/src/fluke_3540/shifts_file.py | 45 ++++++++ python/tests/test_cli_features.py | 84 +++++++++++++++ python/tests/test_shifts.py | 26 +++++ 4 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 python/src/fluke_3540/shifts_file.py diff --git a/python/src/fluke_3540/cli.py b/python/src/fluke_3540/cli.py index 628b7e1..33d20fd 100644 --- a/python/src/fluke_3540/cli.py +++ b/python/src/fluke_3540/cli.py @@ -163,8 +163,23 @@ def build_argparser() -> argparse.ArgumentParser: ap.add_argument("--split-by", dest="split_by", type=str, default=None, metavar="PERIOD", help="Partition the session into time buckets, emitting a full " - "per-bucket report plus a roll-up. PERIOD: hour|day|week or " - "a duration like 30m, 6h, 2d.") + "per-bucket report plus a roll-up. PERIOD: hour|day|week, " + "a duration like 30m, 6h, 2d, OR 'shifts' for named " + "shift windows (see --shifts / --shifts-file).") + + # Named, configurable shift windows (--split-by shifts) + ap.add_argument("--shifts", dest="shifts", type=str, default=None, + metavar="SPEC", + help="Shift windows as 'name=HH:MM-HH:MM,...' (comma-separated). " + "A window where end<=start wraps past midnight " + "(e.g. night=18:00-06:00). Interpreted in --tz (UTC if " + "unset). Default day=06:00-18:00,night=18:00-06:00. " + "Requires --split-by shifts.") + ap.add_argument("--shifts-file", dest="shifts_file", type=Path, default=None, + metavar="FILE", + help="JSON file of shift windows " + "({\"shifts\":[{\"name\",\"start\",\"end\"}]}); an " + "alternative to --shifts. See docs/SHIFTS.md.") # Event markers / correlation ap.add_argument("--mark", action="append", default=None, metavar="ISO=LABEL", @@ -592,6 +607,113 @@ def _run_split_by(args: argparse.Namespace, outdir: Path, store: ColumnStore, f"per-bucket reports under {outdir}/