diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml new file mode 100644 index 0000000..cc1083b --- /dev/null +++ b/.github/workflows/freshness.yml @@ -0,0 +1,25 @@ +name: Freshness + +on: + schedule: + - cron: "47 9 2 * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check-demo-date: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + # The checker imports only stdlib-backed constants, so no dependency install is needed. + - name: Check demo-data freshness + run: python scripts/check_freshness.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c7bc927..31accca 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -108,12 +108,12 @@ their output contract contains aggregate dimensions and measures only. ## Trust controls -- Synthetic markers and reserved identifiers/domains are validated. -- The test harness blocks socket access. -- Do Not Contact removes an organization from outreach priority. -- Drafts and reports are local, synthetic, and human-reviewed. -- No page can obtain applicant-level output through a role or navigation path. -- Generated artifacts are reproducible but excluded from version control. +- Synthetic markers and reserved identifiers/domains are validated. +- The test harness blocks socket access. +- Do Not Contact removes an organization from outreach priority. +- Drafts and reports are local, synthetic, and human-reviewed. +- No page can obtain applicant-level output through a role or navigation path. +- Generated artifacts are reproducible but excluded from version control. ## Production transition diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py new file mode 100644 index 0000000..684f34d --- /dev/null +++ b/scripts/check_freshness.py @@ -0,0 +1,81 @@ +"""Scheduled freshness runbook. +Reads: DATA_AS_OF_DATE from src/tens_hq/constants.py. +Red: the synthetic demo date is more than 365 days old or the fact registry is empty. +Remediation: review the demo facts, regenerate them, and advance DATA_AS_OF_DATE. +Notification: GitHub emails the scheduled-run actor; 60 days of inactivity may auto-disable the schedule with warning. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from datetime import date, timedelta +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from tens_hq.constants import DATA_AS_OF_DATE # noqa: E402 + + +MAX_FACT_AGE_DAYS = 365 + + +@dataclass(frozen=True) +class Fact: + name: str + as_of: date + remediation: str + + @property + def deadline(self) -> date: + return self.as_of + timedelta(days=MAX_FACT_AGE_DAYS) + + +def fact_registry() -> tuple[Fact, ...]: + return ( + Fact( + name="DATA_AS_OF_DATE", + as_of=DATA_AS_OF_DATE, + remediation="review the demo facts, regenerate them, and advance DATA_AS_OF_DATE", + ), + ) + + +def stale_facts(today: date) -> list[str]: + facts = fact_registry() + if not facts: + return ["fact registry is empty; restore at least one machine-readable dated fact"] + return [ + f"{fact.name}: deadline {fact.deadline.isoformat()}; {fact.remediation}" + for fact in facts + if today > fact.deadline + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Check ROCC's dated demo facts for staleness.") + parser.add_argument("--today", type=date.fromisoformat, help="override today with YYYY-MM-DD") + args = parser.parse_args() + today = args.today or date.today() + + facts = fact_registry() + findings = stale_facts(today) + if not facts: + return 1 + + for finding in findings: + print(f"STALE: {finding}") + + for fact in facts: + status = "STALE" if today > fact.deadline else "FRESH" + print( + f"{status}: {fact.name} as of {fact.as_of.isoformat()} " + f"(deadline {fact.deadline.isoformat()})" + ) + return 1 if findings else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/tens_hq/pages.py b/src/tens_hq/pages.py index 41c2bc2..211eec7 100644 --- a/src/tens_hq/pages.py +++ b/src/tens_hq/pages.py @@ -344,7 +344,10 @@ def render_site_readiness(data: DemoData, target: float, scenario: str) -> None: cols[1].metric("90-day", f"{row90['projected_ratio']:.1%}", f"{row90['direction']:+.1%}") cols[2].metric("180-day", f"{row180['projected_ratio']:.1%}") cols[3].metric("Open roles", int(row90["open_roles_count"])) - cols[4].metric("Ready hires still needed", int(row90["qualified_hiring_need"] or 0)) + cols[4].metric( + "Ready hires still needed", + int(0 if pd.isna(row90["qualified_hiring_need"]) else row90["qualified_hiring_need"]), + ) cols[5].metric("Projected pipeline arrivals", f"{row90['expected_ready_hires']:.1f}") st.markdown(f'
{html.escape(row90["explanation"])}
', unsafe_allow_html=True) _cov = row90["pipeline_coverage"] diff --git a/src/tens_hq/services.py b/src/tens_hq/services.py index 25bfa1b..eade88b 100644 --- a/src/tens_hq/services.py +++ b/src/tens_hq/services.py @@ -123,7 +123,7 @@ def leadership_summary_markdown(forecasts: pd.DataFrame, queue: pd.DataFrame) -> 3, "qualified_hiring_need" ) risks = "\n".join( - f"- {row.site_name}: {row.risk_status}, projected {row.projected_ratio:.1%}, need {int(row.qualified_hiring_need or 0)}" + f"- {row.site_name}: {row.risk_status}, projected {row.projected_ratio:.1%}, need {int(0 if pd.isna(row.qualified_hiring_need) else row.qualified_hiring_need)}" for row in risk_sites.itertuples(index=False) ) or "- No At Risk or Critical sites in the selected scenario." actions = "\n".join( diff --git a/tests/test_freshness_check.py b/tests/test_freshness_check.py new file mode 100644 index 0000000..786325b --- /dev/null +++ b/tests/test_freshness_check.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from datetime import date +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import check_freshness # noqa: E402 + + +def test_fresh_on_2026_07_27(): + assert check_freshness.stale_facts(date(2026, 7, 27)) == [] + + +def test_stale_after_365_days(): + findings = check_freshness.stale_facts(date(2027, 7, 1)) + assert len(findings) == 1 + assert "DATA_AS_OF_DATE" in findings[0] + assert "deadline 2027-06-30" in findings[0] + + +def test_deadline_day_is_fresh(): + assert check_freshness.stale_facts(date(2027, 6, 30)) == [] + + +def test_stale_main_prints_remediation(monkeypatch, capsys): + monkeypatch.setattr( + sys, "argv", ["check_freshness.py", "--today", "2027-07-01"] + ) + + assert check_freshness.main() == 1 + assert ( + "review the demo facts, regenerate them, and advance DATA_AS_OF_DATE" + in capsys.readouterr().out + ) + + +def test_registry_is_not_empty(): + assert check_freshness.fact_registry() diff --git a/tests/test_services.py b/tests/test_services.py new file mode 100644 index 0000000..3b10284 --- /dev/null +++ b/tests/test_services.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from dataclasses import replace +import inspect + +import pytest + +from tens_hq import pages +import tens_hq.services as services +from tens_hq.metrics import forecast_sites, source_performance +from tens_hq.services import ( + build_outreach_queue, + leadership_summary_markdown, + outreach_email_draft, + site_report_markdown, +) + + +def test_do_not_contact_orgs_are_excluded_from_outreach_queue(demo_data): + do_not_contact = demo_data.organizations["relationship_status"].eq("Do Not Contact") + assert do_not_contact.sum() > 0 + + forecasts = forecast_sites(demo_data) + scores = source_performance(demo_data) + queue = build_outreach_queue(demo_data, scores, forecasts) + + assert not queue["relationship_status"].eq("Do Not Contact").any() + + +def test_do_not_contact_partner_priority_score_is_zero(demo_data): + scored = source_performance(demo_data) + organization_id = scored.iloc[0]["organization_id"] + assert scored.iloc[0]["partner_priority_score"] != 0 + + organizations = demo_data.organizations.copy() + organizations.loc[ + organizations["organization_id"].eq(organization_id), "relationship_status" + ] = "Do Not Contact" + + scores = source_performance(replace(demo_data, organizations=organizations)) + do_not_contact = scores.loc[scores["organization_id"].eq(organization_id)] + + assert len(do_not_contact) == 1 + assert do_not_contact.iloc[0]["partner_priority_score"] == 0 + + +def test_outreach_drafts_are_synthetic_and_carry_no_delivery_capability(): + warm_subject, warm_body = outreach_email_draft( + "Synthetic Partner", + "Taylor Example", + "Synthetic Site", + "Administrative Support", + "Warm", + ) + cold_subject, cold_body = outreach_email_draft( + "Synthetic Partner", + "Taylor Example", + "Synthetic Site", + "Administrative Support", + "Cold", + ) + + assert warm_subject + assert cold_subject + assert "ROCC Demo User (Synthetic)" in warm_body + assert "ROCC Demo User (Synthetic)" in cold_body + assert "has not been sent" in cold_body + + source = inspect.getsource(services) + assert "smtplib" not in source + assert "requests" not in source + assert ".send(" not in source + assert "sendmail" not in source + + +def test_reports_carry_decision_boundary_banners(demo_data): + forecasts = forecast_sites(demo_data) + scores = source_performance(demo_data) + queue = build_outreach_queue(demo_data, scores, forecasts) + + site_report = site_report_markdown(forecasts.iloc[0]) + leadership_summary = leadership_summary_markdown(forecasts, queue) + + assert "NOT AN OFFICIAL ODLH COMPLIANCE DETERMINATION" in site_report + assert "SYNTHETIC DEMO DATA — NOT FOR EMPLOYMENT OR COMPLIANCE DECISIONS" in leadership_summary + + +@pytest.mark.parametrize("missing_need", [None, float("nan")], ids=["none", "nan"]) +def test_missing_qualified_hiring_need_is_zero_in_frame_consumers( + monkeypatch, demo_data, missing_need +): + forecasts90 = forecast_sites(demo_data) + needs = forecasts90["qualified_hiring_need"].astype(float).tolist() + needs[0] = missing_need + forecasts90 = forecasts90.assign(qualified_hiring_need=needs) + forecasts180 = forecast_sites(demo_data, horizon_days=180) + + assert str(forecasts90["qualified_hiring_need"].dtype) == "float64" + + scores = source_performance(demo_data) + queue = build_outreach_queue(demo_data, scores, forecasts90) + leadership_summary = leadership_summary_markdown(forecasts90, queue) + + assert "North Harbor Services: At Risk" in leadership_summary + assert "need 0" in leadership_summary + + rendered_metrics = {} + + class MetricColumn: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def metric(self, label, value, *_args, **_kwargs): + rendered_metrics[label] = value + + monkeypatch.setattr( + pages, + "_cached_forecast_sites", + lambda _data, _seed, _target, _scenario, horizon: ( + forecasts90 if horizon == 90 else forecasts180 + ), + ) + monkeypatch.setattr(pages, "_cached_source_performance", lambda *_args: scores) + monkeypatch.setattr( + pages.st, + "columns", + lambda spec, *_args, **_kwargs: [ + MetricColumn() for _ in range(spec if isinstance(spec, int) else len(spec)) + ], + ) + monkeypatch.setattr(pages.st, "selectbox", lambda _label, options: options[0]) + monkeypatch.setattr(pages.st, "markdown", lambda *_args, **_kwargs: None) + monkeypatch.setattr(pages.st, "caption", lambda *_args, **_kwargs: None) + monkeypatch.setattr(pages.st, "plotly_chart", lambda *_args, **_kwargs: None) + monkeypatch.setattr(pages.st, "dataframe", lambda *_args, **_kwargs: None) + monkeypatch.setattr(pages.st, "info", lambda *_args, **_kwargs: None) + + pages.render_site_readiness(demo_data, target=0.75, scenario="Base") + + assert rendered_metrics["Ready hires still needed"] == 0 diff --git a/tests/test_trust_controls.py b/tests/test_trust_controls.py new file mode 100644 index 0000000..62e93b5 --- /dev/null +++ b/tests/test_trust_controls.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import ast +from pathlib import Path +import re + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +ARCHITECTURE = ROOT / "docs" / "ARCHITECTURE.md" +CONTROL_TEST_MAP: dict[str, tuple[str, str]] = { + "synthetic-markers-validated": ( + "test_validation.py", + "test_validation_flags_each_corrupted_contract", + ), + "socket-guard": ("test_socket_guard.py", "test_direct_socket_connect_is_blocked"), + "do-not-contact-override": ( + "test_services.py", + "test_do_not_contact_orgs_are_excluded_from_outreach_queue", + ), + "drafts-human-reviewed": ( + "test_services.py", + "test_outreach_drafts_are_synthetic_and_carry_no_delivery_capability", + ), + "no-applicant-level-output": ( + "test_ui_contract.py", + "test_pipeline_health_never_renders_applicant_identifiers_or_names", + ), + "generated-artifacts-untracked": ( + "test_trust_controls.py", + "test_generated_artifacts_directory_is_gitignored", + ), +} + + +def _extract_trust_controls() -> tuple[bool, set[str], list[str], list[str]]: + text = ARCHITECTURE.read_text(encoding="utf-8") + match = re.search(r"^## Trust controls\s*$([\s\S]*?)(?=^##\s|\Z)", text, re.MULTILINE) + if match is None: + return False, set(), [], [] + + section = match.group(1) + bullet_lines = [line for line in section.splitlines() if line.startswith("- ")] + annotated_bullets = [ + line + for line in bullet_lines + if re.search(r"", line) + ] + control_ids = set(re.findall(r"", section)) + return True, control_ids, bullet_lines, annotated_bullets + + +def test_trust_controls_sweep_covers_every_documented_control(): + heading_found, control_ids, bullet_lines, annotated_bullets = _extract_trust_controls() + unannotated_bullets = [ + line for line in bullet_lines if line not in annotated_bullets + ] + + assert heading_found + assert len(bullet_lines) == len(annotated_bullets), ( + "Every bullet in the Trust controls section must carry a control annotation; " + f"unannotated bullets: {unannotated_bullets}" + ) + assert control_ids == set(CONTROL_TEST_MAP) + assert len(control_ids) >= 6 + assert {"do-not-contact-override", "socket-guard"} <= control_ids + + +@pytest.mark.parametrize("control_id", sorted(CONTROL_TEST_MAP)) +def test_control_maps_to_an_existing_test(control_id): + filename, function_name = CONTROL_TEST_MAP[control_id] + tree = ast.parse((ROOT / "tests" / filename).read_text(encoding="utf-8")) + + assert any( + isinstance(node, ast.FunctionDef) and node.name == function_name + for node in ast.walk(tree) + ) + + +def test_generated_artifacts_directory_is_gitignored(): + ignored_lines = (ROOT / ".gitignore").read_text(encoding="utf-8").splitlines() + assert "data/generated/" in ignored_lines diff --git a/tests/test_validation.py b/tests/test_validation.py index 3955729..cf475cc 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,9 +1,123 @@ from __future__ import annotations +from dataclasses import replace + +import pytest + from tens_hq.constants import COUNTY_NAMES, SITE_PROFILES from tens_hq.validation import expected_row_counts, validate_demo_data +def _with_frame(data, name, mutate): + frame = getattr(data, name).copy() + mutate(frame) + return replace(data, **{name: frame}) + + +def _wrong_row_count(data): + return replace(data, counties=data.counties.iloc[:-1].copy()) + + +def _prohibited_column(data): + return _with_frame(data, "sites", lambda frame: frame.__setitem__("medical_record", "none")) + + +def _missing_synthetic_flag(data): + return replace(data, sites=data.sites.drop(columns="synthetic_flag").copy()) + + +def _unmarked_synthetic_row(data): + return _with_frame(data, "sites", lambda frame: frame.__setitem__("synthetic_flag", False)) + + +def _invalid_id_prefix(data): + return _with_frame(data, "sites", lambda frame: frame.__setitem__("site_id", "INVALID")) + + +def _duplicate_id(data): + def mutate(frame): + frame.loc[frame.index[1], "site_id"] = frame.loc[frame.index[0], "site_id"] + + return _with_frame(data, "sites", mutate) + + +def _non_reserved_email(data): + return _with_frame( + data, + "contacts", + lambda frame: frame.__setitem__("contact_email", "person@example.com"), + ) + + +def _non_reserved_website(data): + return _with_frame( + data, + "organizations", + lambda frame: frame.__setitem__("website_url", "https://example.com"), + ) + + +def _qdl_exceeds_total(data): + def mutate(frame): + frame.loc[frame.index[0], "mock_qdl_hours"] = frame.loc[frame.index[0], "total_direct_labor_hours"] + 1 + + return _with_frame(data, "labor_hours", mutate) + + +def _negative_hours(data): + def mutate(frame): + frame.loc[frame.index[0], "mock_qdl_hours"] = -1 + + return _with_frame(data, "labor_hours", mutate) + + +def _ratio_does_not_reproduce(data): + def mutate(frame): + frame.loc[frame.index[0], "current_ratio_pct"] += 1 + + return _with_frame(data, "labor_hours", mutate) + + +def _missing_synthetic_display_label(data): + return _with_frame( + data, + "applicants", + lambda frame: frame.__setitem__("display_label", "Applicant"), + ) + + +def _eligibility_before_start(data): + def mutate(frame): + index = frame.index[frame["start_date"].isna()][0] + frame.loc[index, "eligibility_review_status"] = "Cleared" + + return _with_frame(data, "applicants", mutate) + + +def _documentation_before_start(data): + def mutate(frame): + index = frame.index[frame["start_date"].isna()][0] + frame.loc[index, "documentation_status"] = "Complete" + + return _with_frame(data, "applicants", mutate) + + +def _unknown_source_organization(data): + return _with_frame( + data, + "applicants", + lambda frame: frame.__setitem__("source_organization_id", "SYN-ORG-UNKNOWN"), + ) + + +def _unknown_target_site(data): + return _with_frame( + data, + "applicants", + lambda frame: frame.__setitem__("target_site_id", "SYN-SITE-UNKNOWN"), + ) + + def test_derivable_counts_track_generator_constants(): counts = expected_row_counts() assert counts["sites"] == len(SITE_PROFILES) == 12 @@ -16,3 +130,38 @@ def test_expected_counts_reproduce_actual_generation(demo_data): for name, expected in counts.items(): assert len(getattr(demo_data, name)) == expected, name assert validate_demo_data(demo_data).ok + + +@pytest.mark.parametrize( + ("mutator", "error"), + [ + (_wrong_row_count, "counties: expected 48 rows, found 47"), + (_prohibited_column, "prohibited columns present"), + (_missing_synthetic_flag, "synthetic_flag is missing"), + (_unmarked_synthetic_row, "one or more rows are not marked synthetic"), + (_invalid_id_prefix, "ID prefix contract failed"), + (_duplicate_id, "duplicate IDs detected"), + (_non_reserved_email, "non-reserved email domain detected"), + (_non_reserved_website, "non-reserved website domain detected"), + (_qdl_exceeds_total, "QDL hours exceed total direct labor hours"), + (_negative_hours, "negative hours detected"), + (_ratio_does_not_reproduce, "stored ratios do not reproduce"), + (_missing_synthetic_display_label, "synthetic display label missing"), + (_eligibility_before_start, "eligibility status used before the synthetic start-stage gate"), + (_documentation_before_start, "documentation status used before the synthetic start-stage gate"), + (_unknown_source_organization, "unknown source organization reference"), + (_unknown_target_site, "unknown target site reference"), + ], + ids=lambda value: value.__name__ if callable(value) else None, +) +def test_validation_flags_each_corrupted_contract(demo_data, mutator, error): + result = validate_demo_data(mutator(demo_data)) + assert result.ok is False + assert any(error in message for message in result.errors) + + +def test_thin_stage_history_is_a_warning_not_an_error(demo_data): + corrupted = replace(demo_data, stage_history=demo_data.stage_history.iloc[:3999].copy()) + result = validate_demo_data(corrupted) + assert result.ok is True + assert "stage_history: fewer than 4,000 events; dashboard story may be thin" in result.warnings