From 9f017c251716277fda596a534f351891fc2080ea Mon Sep 17 00:00:00 2001 From: Sam-24-dev Date: Mon, 31 Aug 2026 21:33:48 +0100 Subject: [PATCH 1/5] fix(ci): harden weekly ETL schedule --- .github/workflows/etl_semanal.yml | 5 +- README.md | 6 +- scripts/check_source_freshness.py | 101 +++++++++++++++++++++++++++ tests/test_check_source_freshness.py | 61 ++++++++++++++++ tests/test_workflow_etl_contract.py | 11 +++ 5 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 scripts/check_source_freshness.py create mode 100644 tests/test_check_source_freshness.py diff --git a/.github/workflows/etl_semanal.yml b/.github/workflows/etl_semanal.yml index 8bd0b61..8a49fbe 100644 --- a/.github/workflows/etl_semanal.yml +++ b/.github/workflows/etl_semanal.yml @@ -2,7 +2,7 @@ name: ETL Weekly Data Refresh on: schedule: - - cron: "0 8 * * 1" + - cron: "17 8 * * 1" workflow_dispatch: permissions: @@ -502,6 +502,9 @@ jobs: --project-root . \ --expect-previous-history "${{ steps.previous_history.outputs.expect_previous_history }}" + - name: Enforce canonical source freshness guard + run: python scripts/check_source_freshness.py --project-root . --max-source-age-hours 192 + - name: Upload aggregate artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: diff --git a/README.md b/README.md index a2f54c4..470f809 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ flutter run -d chrome ## Automation (GitHub Actions) 1. **ETL Weekly Refresh** (`etl_semanal.yml`) - - Schedule: Monday `08:00 UTC` + - Schedule: Monday `08:17 UTC` (approximately 03:17 America/Guayaquil). - Runs all ETLs, Trend Score, syncs assets, validates contracts, and publishes data. 2. **Dependency Security** (`dependency_security.yml`) - Schedule: Monday `09:00 UTC` @@ -92,6 +92,10 @@ flutter run -d chrome 3. **Deploy Frontend** (`deploy_frontend.yml`) - Publishes Flutter Web to GitHub Pages. +### Delayed weekly run recovery + +GitHub Actions schedules can be delayed. First inspect **ETL Weekly Data Refresh** in the Actions tab and confirm that no run is in progress for the `etl-pipeline` concurrency group. If the scheduled run is absent or has failed, use **Run workflow** on `main` only after that check. Review the run summary and the source freshness guard before considering the published data healthy. + --- ## Project Structure diff --git a/scripts/check_source_freshness.py b/scripts/check_source_freshness.py new file mode 100644 index 0000000..cbbe153 --- /dev/null +++ b/scripts/check_source_freshness.py @@ -0,0 +1,101 @@ +"""Reject canonical source snapshots that are too old to publish.""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path + + +DEFAULT_MAX_SOURCE_AGE_HOURS = 192 +SOURCE_DATASET_PREFIXES = { + "github": "github_", + "stackoverflow": "so_", + "reddit": "reddit_", +} + + +def _parse_utc_timestamp(value: object) -> datetime | None: + if not isinstance(value, str) or not value.endswith("Z"): + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError: + return None + + +def _latest_source_timestamp(dataset_summaries: object, prefix: str) -> tuple[str, datetime] | None: + if not isinstance(dataset_summaries, list): + return None + + latest: tuple[str, datetime] | None = None + for summary in dataset_summaries: + if not isinstance(summary, dict) or not str(summary.get("dataset", "")).startswith(prefix): + continue + timestamp = _parse_utc_timestamp(summary.get("updated_at_utc")) + if timestamp is None: + continue + raw_timestamp = str(summary["updated_at_utc"]) + if latest is None or timestamp > latest[1]: + latest = (raw_timestamp, timestamp) + return latest + + +def check_source_freshness( + project_root: Path | str, + *, + max_source_age_hours: int = DEFAULT_MAX_SOURCE_AGE_HOURS, +) -> dict[str, object]: + """Validate source timestamps against the manifest generation time.""" + manifest_path = Path(project_root) / "frontend" / "assets" / "data" / "run_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if not isinstance(manifest, dict): + raise ValueError("Source freshness unavailable: run manifest must be a JSON object") + + reference_at = _parse_utc_timestamp(manifest.get("generated_at_utc")) + if reference_at is None: + raise ValueError("Source freshness unavailable: manifest generated_at_utc is invalid") + + source_updated_at_utc: dict[str, str] = {} + errors: list[str] = [] + for source, prefix in SOURCE_DATASET_PREFIXES.items(): + latest = _latest_source_timestamp(manifest.get("dataset_summaries"), prefix) + if latest is None: + errors.append(f"Source freshness unavailable: {source} has no valid canonical updated_at_utc") + continue + + updated_at_raw, updated_at = latest + source_updated_at_utc[source] = updated_at_raw + age_hours = (reference_at - updated_at).total_seconds() / 3600 + if age_hours > max_source_age_hours: + errors.append( + f"Source freshness stale: {source} is {age_hours:.1f}h old " + f"(maximum {max_source_age_hours}h)" + ) + + if errors: + raise ValueError("; ".join(errors)) + + return { + "max_source_age_hours": max_source_age_hours, + "source_updated_at_utc": source_updated_at_utc, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-root", type=Path, default=Path(".")) + parser.add_argument("--max-source-age-hours", type=int, default=DEFAULT_MAX_SOURCE_AGE_HOURS) + args = parser.parse_args() + + try: + check_source_freshness(args.project_root, max_source_age_hours=args.max_source_age_hours) + except ValueError as error: + print(f"Source freshness guard failed: {error}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_check_source_freshness.py b/tests/test_check_source_freshness.py new file mode 100644 index 0000000..df77733 --- /dev/null +++ b/tests/test_check_source_freshness.py @@ -0,0 +1,61 @@ +import json + +import pytest + +from scripts.check_source_freshness import check_source_freshness + + +def _write_manifest(tmp_path, *, github, stackoverflow, reddit): + assets_root = tmp_path / "frontend" / "assets" / "data" + assets_root.mkdir(parents=True) + payload = { + "generated_at_utc": "2026-08-31T08:17:00Z", + "dataset_summaries": [ + {"dataset": "github_lenguajes", "updated_at_utc": github}, + {"dataset": "so_volumen_preguntas", "updated_at_utc": stackoverflow}, + {"dataset": "reddit_temas_emergentes", "updated_at_utc": reddit}, + {"dataset": "trend_score", "updated_at_utc": "2026-08-31T08:17:00Z"}, + ], + } + (assets_root / "run_manifest.json").write_text(json.dumps(payload), encoding="utf-8") + + +def test_allows_mixed_source_freshness_within_weekly_grace(tmp_path): + _write_manifest( + tmp_path, + github="2026-08-24T08:17:00Z", + stackoverflow="2026-08-24T08:17:00Z", + reddit="2026-08-31T08:16:00Z", + ) + + result = check_source_freshness(tmp_path) + + assert result["source_updated_at_utc"] == { + "github": "2026-08-24T08:17:00Z", + "stackoverflow": "2026-08-24T08:17:00Z", + "reddit": "2026-08-31T08:16:00Z", + } + + +def test_rejects_source_older_than_weekly_grace(tmp_path): + _write_manifest( + tmp_path, + github="2026-08-23T08:16:59Z", + stackoverflow="2026-08-31T08:17:00Z", + reddit="2026-08-31T08:17:00Z", + ) + + with pytest.raises(ValueError, match="Source freshness stale: github"): + check_source_freshness(tmp_path) + + +def test_rejects_missing_source_timestamp_without_manifest_fallback(tmp_path): + _write_manifest( + tmp_path, + github="not-a-timestamp", + stackoverflow="2026-08-31T08:17:00Z", + reddit="2026-08-31T08:17:00Z", + ) + + with pytest.raises(ValueError, match="Source freshness unavailable: github"): + check_source_freshness(tmp_path) diff --git a/tests/test_workflow_etl_contract.py b/tests/test_workflow_etl_contract.py index d34f6e9..d011d45 100644 --- a/tests/test_workflow_etl_contract.py +++ b/tests/test_workflow_etl_contract.py @@ -89,6 +89,17 @@ def test_workflow_no_longer_downloads_nltk_data(): assert "Download NLTK data" not in content +def test_weekly_schedule_avoids_top_of_hour_and_keeps_publish_safeguards(): + content = _load_workflow_text() + + assert 'cron: "17 8 * * 1"' in content + assert "workflow_dispatch:" in content + assert "group: etl-pipeline" in content + assert "cancel-in-progress: false" in content + assert "Enforce canonical source freshness guard" in content + assert "python scripts/check_source_freshness.py --project-root ." in content + + def test_workflow_reddit_job_resets_stale_outputs_and_requires_fresh_latest_files(): content = _load_workflow_text() From 68db562bb6868f6c95caa9f9ccc83075ec787b30 Mon Sep 17 00:00:00 2001 From: Sam-24-dev Date: Mon, 31 Aug 2026 21:48:48 +0100 Subject: [PATCH 2/5] fix(etl): validate each canonical source dataset --- scripts/check_source_freshness.py | 83 +++++++++++++++++++++------- tests/test_check_source_freshness.py | 77 +++++++++++++++++++++----- 2 files changed, 127 insertions(+), 33 deletions(-) diff --git a/scripts/check_source_freshness.py b/scripts/check_source_freshness.py index cbbe153..00e81e2 100644 --- a/scripts/check_source_freshness.py +++ b/scripts/check_source_freshness.py @@ -9,10 +9,24 @@ DEFAULT_MAX_SOURCE_AGE_HOURS = 192 -SOURCE_DATASET_PREFIXES = { - "github": "github_", - "stackoverflow": "so_", - "reddit": "reddit_", +REQUIRED_SOURCE_DATASETS = { + "github": ( + "github_ai_repos_insights", + "github_commits_frameworks", + "github_commits_frameworks_monthly", + "github_correlacion", + "github_lenguajes", + "github_repos_2025", + ), + "stackoverflow": ( + "so_volumen_preguntas", + "so_tasa_aceptacion", + "so_tendencias_mensuales", + ), + "reddit": ( + "reddit_sentimiento_frameworks", + "reddit_temas_emergentes", + ), } @@ -25,21 +39,45 @@ def _parse_utc_timestamp(value: object) -> datetime | None: return None -def _latest_source_timestamp(dataset_summaries: object, prefix: str) -> tuple[str, datetime] | None: +def _oldest_required_timestamp( + dataset_summaries: object, + required_datasets: tuple[str, ...], +) -> tuple[str, datetime] | None: if not isinstance(dataset_summaries, list): return None - latest: tuple[str, datetime] | None = None - for summary in dataset_summaries: - if not isinstance(summary, dict) or not str(summary.get("dataset", "")).startswith(prefix): - continue - timestamp = _parse_utc_timestamp(summary.get("updated_at_utc")) + summaries = { + str(summary.get("dataset")): summary + for summary in dataset_summaries + if isinstance(summary, dict) + } + timestamps: list[tuple[str, datetime]] = [] + for dataset in required_datasets: + summary = summaries.get(dataset) + timestamp = _parse_utc_timestamp(summary.get("updated_at_utc")) if summary else None if timestamp is None: - continue - raw_timestamp = str(summary["updated_at_utc"]) - if latest is None or timestamp > latest[1]: - latest = (raw_timestamp, timestamp) - return latest + return None + timestamps.append((str(summary["updated_at_utc"]), timestamp)) + + return min(timestamps, key=lambda item: item[1]) + + +def _source_error(dataset_summaries: object, source: str, required_datasets: tuple[str, ...]) -> str | None: + if not isinstance(dataset_summaries, list): + return f"Source freshness unavailable: {source} has no canonical dataset summaries" + + summaries = { + str(summary.get("dataset")): summary + for summary in dataset_summaries + if isinstance(summary, dict) + } + for dataset in required_datasets: + summary = summaries.get(dataset) + if summary is None: + return f"Source freshness unavailable: {source} is missing required dataset {dataset}" + if _parse_utc_timestamp(summary.get("updated_at_utc")) is None: + return f"Source freshness unavailable: {source} has invalid updated_at_utc for {dataset}" + return None def check_source_freshness( @@ -59,13 +97,18 @@ def check_source_freshness( source_updated_at_utc: dict[str, str] = {} errors: list[str] = [] - for source, prefix in SOURCE_DATASET_PREFIXES.items(): - latest = _latest_source_timestamp(manifest.get("dataset_summaries"), prefix) - if latest is None: - errors.append(f"Source freshness unavailable: {source} has no valid canonical updated_at_utc") + dataset_summaries = manifest.get("dataset_summaries") + for source, required_datasets in REQUIRED_SOURCE_DATASETS.items(): + source_error = _source_error(dataset_summaries, source, required_datasets) + if source_error is not None: + errors.append(source_error) continue - updated_at_raw, updated_at = latest + oldest = _oldest_required_timestamp(dataset_summaries, required_datasets) + if oldest is None: + errors.append(f"Source freshness unavailable: {source} has no valid canonical updated_at_utc") + continue + updated_at_raw, updated_at = oldest source_updated_at_utc[source] = updated_at_raw age_hours = (reference_at - updated_at).total_seconds() / 3600 if age_hours > max_source_age_hours: diff --git a/tests/test_check_source_freshness.py b/tests/test_check_source_freshness.py index df77733..76d637a 100644 --- a/tests/test_check_source_freshness.py +++ b/tests/test_check_source_freshness.py @@ -5,15 +5,44 @@ from scripts.check_source_freshness import check_source_freshness -def _write_manifest(tmp_path, *, github, stackoverflow, reddit): +CANONICAL_DATASETS = { + "github": ( + "github_ai_repos_insights", + "github_commits_frameworks", + "github_commits_frameworks_monthly", + "github_correlacion", + "github_lenguajes", + "github_repos_2025", + ), + "stackoverflow": ( + "so_volumen_preguntas", + "so_tasa_aceptacion", + "so_tendencias_mensuales", + ), + "reddit": ( + "reddit_sentimiento_frameworks", + "reddit_temas_emergentes", + ), +} + + +def _write_manifest(tmp_path, *, updated_at_by_dataset=None, missing_datasets=()): assets_root = tmp_path / "frontend" / "assets" / "data" assets_root.mkdir(parents=True) + timestamps = { + dataset: "2026-08-31T08:17:00Z" + for datasets in CANONICAL_DATASETS.values() + for dataset in datasets + } + timestamps.update(updated_at_by_dataset or {}) payload = { "generated_at_utc": "2026-08-31T08:17:00Z", "dataset_summaries": [ - {"dataset": "github_lenguajes", "updated_at_utc": github}, - {"dataset": "so_volumen_preguntas", "updated_at_utc": stackoverflow}, - {"dataset": "reddit_temas_emergentes", "updated_at_utc": reddit}, + {"dataset": dataset, "updated_at_utc": updated_at} + for dataset, updated_at in timestamps.items() + if dataset not in missing_datasets + ] + + [ {"dataset": "trend_score", "updated_at_utc": "2026-08-31T08:17:00Z"}, ], } @@ -23,9 +52,15 @@ def _write_manifest(tmp_path, *, github, stackoverflow, reddit): def test_allows_mixed_source_freshness_within_weekly_grace(tmp_path): _write_manifest( tmp_path, - github="2026-08-24T08:17:00Z", - stackoverflow="2026-08-24T08:17:00Z", - reddit="2026-08-31T08:16:00Z", + updated_at_by_dataset={ + **{ + dataset: "2026-08-24T08:17:00Z" + for dataset in CANONICAL_DATASETS["github"] + + CANONICAL_DATASETS["stackoverflow"] + }, + "reddit_sentimiento_frameworks": "2026-08-31T08:16:00Z", + "reddit_temas_emergentes": "2026-08-31T08:17:00Z", + }, ) result = check_source_freshness(tmp_path) @@ -40,9 +75,7 @@ def test_allows_mixed_source_freshness_within_weekly_grace(tmp_path): def test_rejects_source_older_than_weekly_grace(tmp_path): _write_manifest( tmp_path, - github="2026-08-23T08:16:59Z", - stackoverflow="2026-08-31T08:17:00Z", - reddit="2026-08-31T08:17:00Z", + updated_at_by_dataset={"github_lenguajes": "2026-08-23T08:16:59Z"}, ) with pytest.raises(ValueError, match="Source freshness stale: github"): @@ -52,10 +85,28 @@ def test_rejects_source_older_than_weekly_grace(tmp_path): def test_rejects_missing_source_timestamp_without_manifest_fallback(tmp_path): _write_manifest( tmp_path, - github="not-a-timestamp", - stackoverflow="2026-08-31T08:17:00Z", - reddit="2026-08-31T08:17:00Z", + updated_at_by_dataset={"github_lenguajes": "not-a-timestamp"}, + ) + + with pytest.raises(ValueError, match="Source freshness unavailable: github"): + check_source_freshness(tmp_path) + + +def test_rejects_stale_required_sibling_even_when_another_github_dataset_is_fresh(tmp_path): + _write_manifest( + tmp_path, + updated_at_by_dataset={ + "github_lenguajes": "2026-08-20T08:17:00Z", + "github_repos_2025": "2026-08-31T08:17:00Z", + }, ) + with pytest.raises(ValueError, match="Source freshness stale: github"): + check_source_freshness(tmp_path) + + +def test_rejects_missing_required_canonical_dataset(tmp_path): + _write_manifest(tmp_path, missing_datasets=("github_lenguajes",)) + with pytest.raises(ValueError, match="Source freshness unavailable: github"): check_source_freshness(tmp_path) From 4df881f236acb463d6dd880342dad296df2aa06d Mon Sep 17 00:00:00 2001 From: Sam-24-dev Date: Mon, 31 Aug 2026 22:15:48 +0100 Subject: [PATCH 3/5] fix(etl): reject future source timestamps --- .github/workflows/etl_semanal.yml | 4 ++++ scripts/check_source_freshness.py | 14 +++++++++++--- tests/test_check_source_freshness.py | 10 ++++++++++ tests/test_workflow_etl_contract.py | 14 ++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/.github/workflows/etl_semanal.yml b/.github/workflows/etl_semanal.yml index 8a49fbe..8e77094 100644 --- a/.github/workflows/etl_semanal.yml +++ b/.github/workflows/etl_semanal.yml @@ -65,16 +65,19 @@ jobs: datos/github_lenguajes.csv datos/github_ai_repos_insights.csv datos/github_commits_frameworks.csv + datos/github_commits_frameworks_monthly.csv datos/github_correlacion.csv datos/latest/github_repos_2025.csv datos/latest/github_lenguajes.csv datos/latest/github_ai_repos_insights.csv datos/latest/github_commits_frameworks.csv + datos/latest/github_commits_frameworks_monthly.csv datos/latest/github_correlacion.csv datos/history/**/github_repos_2025.csv datos/history/**/github_lenguajes.csv datos/history/**/github_ai_repos_insights.csv datos/history/**/github_commits_frameworks.csv + datos/history/**/github_commits_frameworks_monthly.csv datos/history/**/github_correlacion.csv job_stackoverflow: @@ -393,6 +396,7 @@ jobs: github_repos_2025.csv \ github_lenguajes.csv \ github_commits_frameworks.csv \ + github_commits_frameworks_monthly.csv \ github_correlacion.csv \ so_volumen_preguntas.csv \ so_tasa_aceptacion.csv \ diff --git a/scripts/check_source_freshness.py b/scripts/check_source_freshness.py index 00e81e2..edc97c3 100644 --- a/scripts/check_source_freshness.py +++ b/scripts/check_source_freshness.py @@ -62,7 +62,12 @@ def _oldest_required_timestamp( return min(timestamps, key=lambda item: item[1]) -def _source_error(dataset_summaries: object, source: str, required_datasets: tuple[str, ...]) -> str | None: +def _source_error( + dataset_summaries: object, + source: str, + required_datasets: tuple[str, ...], + reference_at: datetime, +) -> str | None: if not isinstance(dataset_summaries, list): return f"Source freshness unavailable: {source} has no canonical dataset summaries" @@ -75,8 +80,11 @@ def _source_error(dataset_summaries: object, source: str, required_datasets: tup summary = summaries.get(dataset) if summary is None: return f"Source freshness unavailable: {source} is missing required dataset {dataset}" - if _parse_utc_timestamp(summary.get("updated_at_utc")) is None: + timestamp = _parse_utc_timestamp(summary.get("updated_at_utc")) + if timestamp is None: return f"Source freshness unavailable: {source} has invalid updated_at_utc for {dataset}" + if timestamp > reference_at: + return f"Source freshness invalid: {source} has future updated_at_utc for {dataset}" return None @@ -99,7 +107,7 @@ def check_source_freshness( errors: list[str] = [] dataset_summaries = manifest.get("dataset_summaries") for source, required_datasets in REQUIRED_SOURCE_DATASETS.items(): - source_error = _source_error(dataset_summaries, source, required_datasets) + source_error = _source_error(dataset_summaries, source, required_datasets, reference_at) if source_error is not None: errors.append(source_error) continue diff --git a/tests/test_check_source_freshness.py b/tests/test_check_source_freshness.py index 76d637a..0436882 100644 --- a/tests/test_check_source_freshness.py +++ b/tests/test_check_source_freshness.py @@ -92,6 +92,16 @@ def test_rejects_missing_source_timestamp_without_manifest_fallback(tmp_path): check_source_freshness(tmp_path) +def test_rejects_required_timestamp_later_than_manifest_generation(tmp_path): + _write_manifest( + tmp_path, + updated_at_by_dataset={"github_lenguajes": "2026-08-31T09:17:00Z"}, + ) + + with pytest.raises(ValueError, match="Source freshness invalid: github"): + check_source_freshness(tmp_path) + + def test_rejects_stale_required_sibling_even_when_another_github_dataset_is_fresh(tmp_path): _write_manifest( tmp_path, diff --git a/tests/test_workflow_etl_contract.py b/tests/test_workflow_etl_contract.py index d011d45..c1897b0 100644 --- a/tests/test_workflow_etl_contract.py +++ b/tests/test_workflow_etl_contract.py @@ -48,6 +48,20 @@ def test_workflow_artifact_handoff_contract_is_defined(): assert "artifact_payload/reddit" in content +def test_workflow_handoffs_required_github_monthly_dataset(): + content = _load_workflow_text() + + for path in ( + "datos/github_commits_frameworks_monthly.csv", + "datos/latest/github_commits_frameworks_monthly.csv", + "datos/history/**/github_commits_frameworks_monthly.csv", + ): + assert path in content + + required_block = content.split("for required in \\", maxsplit=1)[1].split("; do", maxsplit=1)[0] + assert "github_commits_frameworks_monthly.csv" in required_block + + def test_workflow_publish_gate_and_bridge_asset_paths(): content = _load_workflow_text() From 759c27532785dde4cf81b08a12e51f8ae0159d7d Mon Sep 17 00:00:00 2001 From: Sam-24-dev Date: Tue, 1 Sep 2026 01:41:52 +0100 Subject: [PATCH 4/5] fix(etl): reject duplicate canonical summaries --- scripts/check_source_freshness.py | 12 ++++++++++++ tests/test_check_source_freshness.py | 23 +++++++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/scripts/check_source_freshness.py b/scripts/check_source_freshness.py index edc97c3..ac0807c 100644 --- a/scripts/check_source_freshness.py +++ b/scripts/check_source_freshness.py @@ -71,6 +71,18 @@ def _source_error( if not isinstance(dataset_summaries, list): return f"Source freshness unavailable: {source} has no canonical dataset summaries" + seen_datasets: set[str] = set() + required_names = set(required_datasets) + for summary in dataset_summaries: + if not isinstance(summary, dict): + continue + dataset = str(summary.get("dataset")) + if dataset not in required_names: + continue + if dataset in seen_datasets: + return f"Source freshness invalid: {source} has duplicate required dataset {dataset}" + seen_datasets.add(dataset) + summaries = { str(summary.get("dataset")): summary for summary in dataset_summaries diff --git a/tests/test_check_source_freshness.py b/tests/test_check_source_freshness.py index 0436882..7eb9a89 100644 --- a/tests/test_check_source_freshness.py +++ b/tests/test_check_source_freshness.py @@ -26,7 +26,13 @@ } -def _write_manifest(tmp_path, *, updated_at_by_dataset=None, missing_datasets=()): +def _write_manifest( + tmp_path, + *, + updated_at_by_dataset=None, + missing_datasets=(), + prepended_summaries=(), +): assets_root = tmp_path / "frontend" / "assets" / "data" assets_root.mkdir(parents=True) timestamps = { @@ -37,7 +43,8 @@ def _write_manifest(tmp_path, *, updated_at_by_dataset=None, missing_datasets=() timestamps.update(updated_at_by_dataset or {}) payload = { "generated_at_utc": "2026-08-31T08:17:00Z", - "dataset_summaries": [ + "dataset_summaries": list(prepended_summaries) + + [ {"dataset": dataset, "updated_at_utc": updated_at} for dataset, updated_at in timestamps.items() if dataset not in missing_datasets @@ -102,6 +109,18 @@ def test_rejects_required_timestamp_later_than_manifest_generation(tmp_path): check_source_freshness(tmp_path) +def test_rejects_duplicate_required_dataset_even_when_later_entry_is_fresh(tmp_path): + _write_manifest( + tmp_path, + prepended_summaries=( + {"dataset": "github_lenguajes", "updated_at_utc": "2026-08-20T08:17:00Z"}, + ), + ) + + with pytest.raises(ValueError, match="Source freshness invalid: github"): + check_source_freshness(tmp_path) + + def test_rejects_stale_required_sibling_even_when_another_github_dataset_is_fresh(tmp_path): _write_manifest( tmp_path, From d564bded57c4dc322039ac8cc86cc2d8191a8ddd Mon Sep 17 00:00:00 2001 From: Sam-24-dev Date: Tue, 1 Sep 2026 02:26:22 +0100 Subject: [PATCH 5/5] fix(etl): verify github insights handoff --- .github/workflows/etl_semanal.yml | 1 + tests/test_workflow_etl_contract.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/etl_semanal.yml b/.github/workflows/etl_semanal.yml index 8e77094..06bc13e 100644 --- a/.github/workflows/etl_semanal.yml +++ b/.github/workflows/etl_semanal.yml @@ -395,6 +395,7 @@ jobs: for required in \ github_repos_2025.csv \ github_lenguajes.csv \ + github_ai_repos_insights.csv \ github_commits_frameworks.csv \ github_commits_frameworks_monthly.csv \ github_correlacion.csv \ diff --git a/tests/test_workflow_etl_contract.py b/tests/test_workflow_etl_contract.py index c1897b0..05066ab 100644 --- a/tests/test_workflow_etl_contract.py +++ b/tests/test_workflow_etl_contract.py @@ -62,6 +62,13 @@ def test_workflow_handoffs_required_github_monthly_dataset(): assert "github_commits_frameworks_monthly.csv" in required_block +def test_workflow_handoffs_required_github_ai_insights_dataset(): + content = _load_workflow_text() + + required_block = content.split("for required in \\", maxsplit=1)[1].split("; do", maxsplit=1)[0] + assert "github_ai_repos_insights.csv" in required_block + + def test_workflow_publish_gate_and_bridge_asset_paths(): content = _load_workflow_text()