From 23e1115d7afed6c288a1bf8e59f4994111a4c5cf Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:55:44 +0200 Subject: [PATCH 01/19] feat(data): introduce safe incremental evidence-based stream probes --- scripts/stream_quality.py | 278 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 scripts/stream_quality.py diff --git a/scripts/stream_quality.py b/scripts/stream_quality.py new file mode 100644 index 00000000..3c3c916e --- /dev/null +++ b/scripts/stream_quality.py @@ -0,0 +1,278 @@ +"""Conservative, bounded stream-health evidence for the Radio Atlas crawler. + +A server returning HTTP 200 is not sufficient to call it verified playable. +Only recognizable audio bytes are VERIFIED; HLS manifests remain DEGRADED +until a browser or media segment test is available. No catalog rows are deleted. +""" +import ipaddress +import json +import socket +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone + +SAMPLE_BYTES = 2048 +CHECKS_PER_RUN = 48 +FAILURE_THRESHOLD = 3 + + +def public_stream_url(url, resolver=None): + """Disallow non-global IPs, host credentials and non HTTP(S) schemes.""" + parts = urllib.parse.urlsplit(str(url or "").strip()) + if parts.scheme not in ("http", "https") or not parts.hostname: + raise ValueError("invalid stream URL") + if parts.username or parts.password: + raise ValueError("embedded URL credentials") + port = parts.port + if port is not None and (port < 80 or port > 65535): + raise ValueError("restricted port") + host = parts.hostname.strip(".").lower() + if host == "localhost" or host.endswith((".localhost", ".internal", ".local")): + raise ValueError("local hostname") + try: + addresses = [ipaddress.ip_address(host)] + except ValueError: + lookup = resolver or socket.getaddrinfo + rows = lookup(host, port or (443 if parts.scheme == "https" else 80), + type=socket.SOCK_STREAM) + addresses = [ipaddress.ip_address(x[4][0].split("%")[0]) for x in rows] + if not addresses or any(not ip.is_global for ip in addresses): + raise ValueError("private or unresolved destination") + return parts.geturl() + + +class SafeStreamRedirect(urllib.request.HTTPRedirectHandler): + def __init__(self, resolver=None): + self.resolver = resolver + self.hops = 0 + + def redirect_request(self, req, fp, code, msg, headers, newurl): + self.hops += 1 + if self.hops > 3: + raise ValueError("too many redirects") + target = urllib.parse.urljoin(req.full_url, newurl) + public_stream_url(target, self.resolver) + if req.full_url.startswith("https://") and target.startswith("http://"): + raise ValueError("HTTPS downgrade") + return super().redirect_request(req, fp, code, msg, headers, target) + + +def audio_evidence(sample, content_type="", url=""): + """Return (codec, evidence, contains_audio) using bytes, not status/MIME.""" + data = bytes(sample or b"") + if data.startswith(b"OggS"): + return "OGG", "ogg_header", True + if data.startswith(b"ID3") or ( + len(data) > 1 and data[0] == 255 and data[1] & 0xE0 == 0xE0 + and data[1] not in (0xF1, 0xF9) + ): + return "MP3", "mpeg_frame", True + if len(data) > 1 and data[0] == 255 and data[1] in (0xF1, 0xF9): + return "AAC", "adts_frame", True + if data.startswith(b"RIFF") and data[8:12] == b"WAVE": + return "WAV", "wave_header", True + if data.startswith(b"fLaC"): + return "FLAC", "flac_header", True + if len(data) >= 12 and data[4:8] == b"ftyp" and str(content_type).startswith("audio/"): + return "MP4", "audio_mp4_header", True + if data.lstrip(b"\xef\xbb\xbf \r\n").startswith(b"#EXTM3U"): + lines = data.decode("utf-8", "replace").splitlines() + hls = ".m3u8" in url.split("?", 1)[0].lower() or any( + line.startswith("#EXT-X-") for line in lines + ) + if hls: + marker = any(line.startswith(("#EXT-X-TARGETDURATION", + "#EXT-X-STREAM-INF")) for line in lines) + segment = any(line.strip() and not line.startswith("#") for line in lines) + return "HLS", "hls_manifest" if marker and segment else "invalid_hls", False + return "M3U", "playlist", False + if data.lstrip().lower().startswith(b"[playlist]"): + return "PLS", "playlist", False + return "", "no_audio_signature", False + + +def first_playlist_url(data, base, kind): + lines = data.decode("utf-8", "replace").splitlines() + if kind == "PLS": + lines = [line.split("=", 1)[1].strip() for line in lines + if line.lower().startswith("file") and "=" in line] + for line in lines[:50]: + line = line.strip() + if not line or line.startswith("#"): + continue + url = urllib.parse.urljoin(base, line) + if url != base and url.startswith(("http://", "https://")): + return url + return "" + + +def quality_score(result, official=False): + """0-100: audio 45, TLS 15, browser 15, startup 10, official 15.""" + score = 0 + if result.get("evidence") in ("ogg_header", "mpeg_frame", "adts_frame", + "wave_header", "flac_header", "audio_mp4_header"): + score += 45 + elif result.get("evidence") == "hls_manifest": + score += 15 + if str(result.get("final_url", "")).startswith("https://"): + score += 15 + if result.get("browser_compatible") is True: + score += 15 + if result.get("startup_ms") is not None and result["startup_ms"] < 3000: + score += 10 + if official and result.get("status") in ("VERIFIED", "DEGRADED"): + score += 15 + return score + + +def probe_stream(url, previous=None, official=False, timeout=6, opener=None, + resolver=None, now_iso=None, depth=0): + """Safe limited read (<= 2KiB); one playlist hop; no live-stream drain.""" + previous = previous or {} + now_iso = now_iso or datetime.now(timezone.utc).isoformat(timespec="seconds") + res = {"status": "UNVERIFIED", "http_status": 0, "final_url": url, + "codec": "", "evidence": "", "startup_ms": None, + "browser_compatible": None, "last_checked": now_iso, + "last_successful_verification": previous.get("last_successful_verification"), + "consecutive_failures": int(previous.get("consecutive_failures") or 0)} + start = time.monotonic() + try: + public_stream_url(url, resolver) + client = opener or urllib.request.build_opener(SafeStreamRedirect(resolver)) + request = urllib.request.Request(url, headers={ + "User-Agent": "RadioAtlasHealth/1.0", + "Range": "bytes=0-2047", + "Icy-MetaData": "0", + "Accept-Encoding": "identity", + }) + with client.open(request, timeout=timeout) as response: + res["http_status"] = int(response.getcode() or 0) + final = response.geturl() if hasattr(response, "geturl") else url + public_stream_url(final, resolver) + if url.startswith("https://") and final.startswith("http://"): + raise ValueError("insecure final URL") + res["final_url"] = final + sample = response.read(SAMPLE_BYTES) + res["startup_ms"] = round((time.monotonic() - start) * 1000) + codec, evidence, proven = audio_evidence( + sample, response.headers.get("Content-Type", ""), final + ) + res.update(codec=codec, evidence=evidence) + if not 200 <= res["http_status"] < 300: + raise ValueError("unexpected HTTP status") + if codec in ("M3U", "PLS") and depth == 0: + target = first_playlist_url(sample, final, codec) + if target: + public_stream_url(target, resolver) + if final.startswith("https://") and target.startswith("http://"): + res["evidence"] = "mixed_content_playlist" + else: + nested = probe_stream(target, previous, official, timeout, + opener, resolver, now_iso, 1) + nested["playlist_url"] = url + return nested + if proven: + res["browser_compatible"] = codec in ("MP3", "AAC", "WAV", "MP4") + res["status"] = ("VERIFIED" if res["browser_compatible"] + and final.startswith("https://") else "DEGRADED") + res["last_successful_verification"] = now_iso + res["consecutive_failures"] = 0 + elif evidence == "hls_manifest": + res["status"] = "DEGRADED" + res["consecutive_failures"] = 0 + else: + res["status"] = "UNVERIFIED" + except (ValueError, OSError, urllib.error.URLError, urllib.error.HTTPError) as error: + code = int(getattr(error, "code", 0) or 0) + res["http_status"] = code or res["http_status"] + res["consecutive_failures"] += 1 + res["evidence"] = type(error).__name__ + if code in (401, 403, 405, 429, 451): + res["status"] = "UNVERIFIED" + elif code in (404, 410) and res["consecutive_failures"] >= FAILURE_THRESHOLD: + res["status"] = "BROKEN" + else: + res["status"] = "TEMPORARILY_UNAVAILABLE" + res["quality_score"] = quality_score(res, official) + return res + + +def select_incremental(stations, previous, now_iso, limit=CHECKS_PER_RUN): + """Prioritize curated, previous failures, then rotate across whole world.""" + urls, seen = [], set() + for station in stations: + if not isinstance(station, dict): + continue + url = station.get("url") or "" + if url in seen or not url.startswith(("https://", "http://")): + continue + if station.get("cc", "").upper() == "IL": + continue + if "dailymotion.com" in url or station.get("tokenType") == "easy_b": + continue + seen.add(url) + urls.append(station) + chosen, taken = [], set() + def add(station): + if len(chosen) < limit and station["url"] not in taken: + chosen.append(station) + taken.add(station["url"]) + for row in urls[:132]: + if len(chosen) >= min(16, limit // 3): + break + last = previous.get(row["url"], {}).get("last_checked", "")[:10] + if not last or (datetime.fromisoformat(now_iso[:10]).date() + - datetime.fromisoformat(last).date()).days >= 3: + add(row) + for row in urls: + if len(chosen) >= min(24, limit // 2): + break + info = previous.get(row["url"], {}) + if info.get("status") in ("TEMPORARILY_UNAVAILABLE", "BROKEN") and ( + info.get("last_checked", "")[:10] != now_iso[:10] + ): + add(row) + if urls: + day = datetime.fromisoformat(now_iso[:10]).date().toordinal() + start = (day * 97 * limit) % len(urls) + for i in range(len(urls)): + row = urls[(start + i) % len(urls)] + if previous.get(row["url"], {}).get("last_checked", "")[:10] != now_iso[:10]: + add(row) + if len(chosen) == limit: + break + return chosen + + +def update_quality_report(stations, path, now_iso, limit=CHECKS_PER_RUN, + probe=probe_stream): + """Observational rolling report: never mutates/drops a station catalog.""" + try: + old = json.loads(path.read_text(encoding="utf-8")).get("results", {}) + if not isinstance(old, dict): + old = {} + except (OSError, ValueError): + old = {} + picked = select_incremental(stations, old, now_iso, limit) + results = dict(old) + for row in picked: + results[row["url"]] = probe( + row["url"], old.get(row["url"]), bool(row.get("official") + or "رسمي" in str(row.get("tags", ""))), + now_iso=now_iso, + ) + keys = {s.get("url") for s in stations if isinstance(s, dict)} + results = {key: value for key, value in results.items() if key in keys} + statuses = ("VERIFIED", "DEGRADED", "UNVERIFIED", + "TEMPORARILY_UNAVAILABLE", "BROKEN") + counts = {s: sum(1 for v in results.values() if v.get("status") == s) + for s in statuses} + report = {"updated": now_iso, "catalog_count": len(stations), + "checked_this_run": len(picked), "ever_checked": len(results), + "not_checked": max(0, len(stations) - len(results)), + "status_counts": counts, "results": results} + path.write_text(json.dumps(report, ensure_ascii=False, indent=1), + encoding="utf-8") + return report From fcd2f612a5f2482b947e70c55869d269ba2a5443 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:56:02 +0200 Subject: [PATCH 02/19] fix(data): quarantine large incomplete rebuilds and sample verified audio --- scripts/update_data.py | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/update_data.py b/scripts/update_data.py index 7b521c03..116f9205 100644 --- a/scripts/update_data.py +++ b/scripts/update_data.py @@ -2587,6 +2587,47 @@ def write_station_deltas(stations, previous_rows, now_iso, data_dir=None): }) +def guard_catalog_snapshot(stations, previous_rows): + """Reject sudden incomplete source rebuilds before touching the last good. + + A disappearing directory page must not remove thousands of favorites. + This is a safety gate, not evidence that individual stations are broken. + """ + previous_count = len(previous_rows) + if len(stations) < 10_000: + raise RuntimeError(f"suspiciously small catalog: {len(stations)}") + if previous_count >= 10_000 and len(stations) < int(previous_count * 0.8): + raise RuntimeError( + f"catalog quarantine: {previous_count} -> {len(stations)} " + "(more than 20% lost; preserving the previous snapshot)" + ) + if any(str(s.get("cc") or "").upper() == "IL" for s in stations): + raise RuntimeError("catalog contains excluded country code") + keys = [station_url_key(s.get("url")) for s in stations] + if not all(keys) or len(set(keys)) != len(keys): + raise RuntimeError("invalid or duplicated stream URLs in new snapshot") + + +def write_quality_report(stations, now_iso): + """Optional observational health checks; failures never block the catalog.""" + try: + from stream_quality import update_quality_report + report = update_quality_report( + stations, DATA_DIR / "stream_quality.json", now_iso + ) + print( + "evidence-based stream quality: " + f"{report['checked_this_run']} sampled, " + f"{report['ever_checked']}/{report['catalog_count']} ever checked, " + f"statuses {report['status_counts']}" + ) + except Exception as error: # noqa: BLE001 + print( + f"stream quality probe FAILED: {error} — keeping existing report", + file=sys.stderr, + ) + + def main(): mode = (sys.argv[1] if len(sys.argv) > 1 else "all").lower() if mode not in {"all", "stations", "sdr"}: @@ -2798,6 +2839,7 @@ def main(): if key and key not in seen_urls: seen_urls.add(key) stations.append(station) + guard_catalog_snapshot(stations, previous_rows) print( f"stations: {len(stations)} total " f"(officials {len(officials)} + arabic {len(arabic)} + " @@ -2830,6 +2872,7 @@ def main(): officials + arabic + arab_extra + stations[:CHECK_HEAD_MAX], now_iso, ) + write_quality_report(stations, now_iso) except Exception as e: # noqa: BLE001 print(f"stations FAILED: {e} — preserving last good snapshot", file=sys.stderr) curated = None From f5aad8171ba93e7659e7ca1221756dbb1c332d26 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:56:25 +0200 Subject: [PATCH 03/19] test(data): cover audio evidence, redirects, SSRF, retries and catalog guard --- tests/test_stream_quality.py | 225 +++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 tests/test_stream_quality.py diff --git a/tests/test_stream_quality.py b/tests/test_stream_quality.py new file mode 100644 index 00000000..52e06eea --- /dev/null +++ b/tests/test_stream_quality.py @@ -0,0 +1,225 @@ +"""Regression tests for safe observable stream quality and catalog preservation.""" +import importlib.util +import json +import tempfile +import unittest +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def load_module(name, path): + spec = importlib.util.spec_from_file_location(name, ROOT / path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +health = load_module("radio_stream_quality", "scripts/stream_quality.py") +crawler = load_module("radio_crawler_guard", "scripts/update_data.py") + + +def public_dns(host, port, type=None): + return [(None, None, None, None, ("93.184.216.34", int(port)))] + + +class FakeResponse: + def __init__(self, content=b"ID3\x04\x00\x00", content_type="audio/mpeg", + url="https://radio.example.test/live.mp3", status=200): + self.content = content + self.headers = {"Content-Type": content_type} + self.url = url + self.status = status + self.read_bytes = None + + def getcode(self): + return self.status + + def geturl(self): + return self.url + + def read(self, n=-1): + self.read_bytes = n + return self.content[:n] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class FakeOpener: + def __init__(self, *responses): + self.responses = list(responses) + self.calls = [] + + def open(self, req, timeout=0): + self.calls.append((req.full_url, timeout)) + reply = self.responses.pop(0) + if isinstance(reply, Exception): + raise reply + return reply + + +class StreamSafetyTests(unittest.TestCase): + def test_private_dns_and_local_hosts_are_rejected(self): + for url in ("http://127.0.0.1/", "http://169.254.169.254/", + "http://192.168.1.2/", "http://[::1]/", + "http://localhost/", "file:///etc/passwd", + "http://user:pass@example.test/private"): + with self.subTest(url=url), self.assertRaises(ValueError): + health.public_stream_url(url, public_dns) + + def test_public_https_host_permitted(self): + self.assertEqual( + health.public_stream_url("https://radio.example.test:8443/live", public_dns), + "https://radio.example.test:8443/live", + ) + + def test_redirect_blocks_private_target_and_downgrade(self): + handler = health.SafeStreamRedirect(public_dns) + request = urllib.request.Request("https://radio.example.test/start") + with self.assertRaises(ValueError): + handler.redirect_request(request, None, 302, "", {}, + "http://127.0.0.1/admin") + handler = health.SafeStreamRedirect(public_dns) + with self.assertRaises(ValueError): + handler.redirect_request(request, None, 302, "", {}, + "http://radio.example.test/live") + + def test_redirect_budget_bounded(self): + handler = health.SafeStreamRedirect(public_dns) + request = urllib.request.Request("https://radio.example.test/start") + for _ in range(3): + next_req = handler.redirect_request( + request, None, 302, "", {}, + "https://radio.example.test/live") + self.assertIsNotNone(next_req) + with self.assertRaises(ValueError): + handler.redirect_request(request, None, 302, "", {}, + "https://radio.example.test/live") + + +class StreamEvidenceTests(unittest.TestCase): + URL = "https://radio.example.test/live.mp3" + + def probe(self, *replies, previous=None): + return health.probe_stream(self.URL, previous, opener=FakeOpener(*replies), + resolver=public_dns, + now_iso="2026-11-19T17:35:19+00:00") + + def test_audio_mp3_bytes_verified_not_just_mime(self): + response = FakeResponse() + result = self.probe(response) + self.assertEqual(result["status"], "VERIFIED") + self.assertEqual(result["codec"], "MP3") + self.assertTrue(result["browser_compatible"]) + self.assertEqual(response.read_bytes, health.SAMPLE_BYTES) + self.assertEqual(result["consecutive_failures"], 0) + + def test_audio_aac_and_ogg_bytes_detected(self): + aac = self.probe(FakeResponse(b"\xff\xf1\x50\x80", "audio/aac")) + ogg = self.probe(FakeResponse(b"OggS" + b"\x00" * 32, "audio/ogg")) + self.assertEqual(aac["codec"], "AAC") + self.assertEqual(aac["status"], "VERIFIED") + self.assertEqual(ogg["codec"], "OGG") + self.assertEqual(ogg["status"], "DEGRADED") + + def test_html_and_empty_200_not_verified(self): + for data, mime in [(b"ok", "audio/mpeg"), + (b"", "audio/mpeg"), + (b"ok", "text/html")]: + with self.subTest(data=data, mime=mime): + result = self.probe(FakeResponse(data, mime)) + self.assertEqual(result["status"], "UNVERIFIED") + self.assertEqual(result["evidence"], "no_audio_signature") + + def test_hls_manifest_only_not_called_verified_playable(self): + hls = b"#EXTM3U\n#EXT-X-TARGETDURATION:4\n#EXTINF:4,\nchunk0.ts\n" + result = self.probe(FakeResponse( + hls, "application/vnd.apple.mpegurl", + "https://radio.example.test/stream.m3u8")) + self.assertEqual(result["status"], "DEGRADED") + self.assertEqual(result["evidence"], "hls_manifest") + self.assertIsNone(result["browser_compatible"]) + + def test_playlist_resolves_one_safe_hop(self): + opener = FakeOpener( + FakeResponse(b"#EXTM3U\nhttps://radio.example.test/real.mp3\n", + "audio/x-mpegurl", "https://radio.example.test/list.m3u"), + FakeResponse(b"ID3\x04\x00", "audio/mpeg", + "https://radio.example.test/real.mp3")) + result = health.probe_stream( + "https://radio.example.test/list.m3u", + opener=opener, resolver=public_dns, + now_iso="2026-11-19T17:35:19+00:00") + self.assertEqual(result["status"], "VERIFIED") + self.assertEqual(result["playlist_url"], "https://radio.example.test/list.m3u") + self.assertEqual(len(opener.calls), 2) + + def test_not_found_requires_three_independent_failures(self): + error = lambda: urllib.error.HTTPError(self.URL, 404, "Not Found", {}, None) + first = self.probe(error()) + second = self.probe(error(), previous=first) + third = self.probe(error(), previous=second) + self.assertEqual(first["status"], "TEMPORARILY_UNAVAILABLE") + self.assertEqual(second["status"], "TEMPORARILY_UNAVAILABLE") + self.assertEqual(third["status"], "BROKEN") + + def test_access_denied_does_not_prove_broken(self): + error = urllib.error.HTTPError(self.URL, 403, "Forbidden", {}, None) + result = self.probe(error) + self.assertEqual(result["status"], "UNVERIFIED") + self.assertNotEqual(result["status"], "VERIFIED") + + +class IntegrityTests(unittest.TestCase): + def test_large_catalog_drop_is_quarantined(self): + previous = [{}] * 20000 + fresh = [{"url": "https://radio.example.test/" + str(i), "cc": "EG"} + for i in range(15000)] + with self.assertRaisesRegex(RuntimeError, "quarantine"): + crawler.guard_catalog_snapshot(fresh, previous) + + def test_valid_station_count_passes_and_duplicate_rejected(self): + stations = [{"url": "https://radio.example.test/" + str(i), "cc": "EG"} + for i in range(10000)] + crawler.guard_catalog_snapshot(stations, []) + stations[1] = dict(stations[0]) + with self.assertRaisesRegex(RuntimeError, "duplicated"): + crawler.guard_catalog_snapshot(stations, []) + + def test_incremental_sample_is_unique_and_excludes_IL(self): + rows = [{"url": "https://radio.example.test/" + str(i), "cc": "EG"} + for i in range(100)] + rows += [dict(rows[1]), {"url": "https://radio.example.test/il", "cc": "IL"}] + selected = health.select_incremental(rows, {}, "2026-11-19T17:35:19+00:00", 12) + self.assertEqual(len(selected), 12) + self.assertEqual(len({x["url"] for x in selected}), 12) + self.assertTrue(all(x["cc"] != "IL" for x in selected)) + + def test_quality_report_records_only_sample_not_false_global_success(self): + stations = [{"url": "https://radio.example.test/" + str(i), "cc": "EG"} + for i in range(30)] + immutable = json.dumps(stations) + + def fake_probe(url, previous, official, now_iso): + return {"status": "VERIFIED", "last_checked": now_iso, + "consecutive_failures": 0} + + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "quality.json" + report = health.update_quality_report( + stations, path, "2026-11-19T17:35:19+00:00", + limit=6, probe=fake_probe) + self.assertEqual(report["checked_this_run"], 6) + self.assertEqual(report["status_counts"]["VERIFIED"], 6) + self.assertEqual(report["not_checked"], 24) + self.assertEqual(json.dumps(stations), immutable) + + +if __name__ == "__main__": + unittest.main() From 11712f6e818b981d2fa402930762adf3f78978cc Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:56:56 +0200 Subject: [PATCH 04/19] docs(data): record source audit baseline and evidence limitations --- docs/RADIO_SOURCES_AUDIT_2026-11-19.md | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 docs/RADIO_SOURCES_AUDIT_2026-11-19.md diff --git a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md new file mode 100644 index 00000000..2d182533 --- /dev/null +++ b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md @@ -0,0 +1,78 @@ +# Radio Atlas — Global radio source audit (2026-11-19) + +Baseline from repository tracked assets at 17:35:19 UTC. + +| Metric | Before | Meaning | +| --- | ---: | --- | +| Catalog entries | 60,732 | data/stations.manifest.json, version 34 | +| Historical stream checks | 1,185 | data/stream_checks.json | +| Historical results classified alive | 1,071 (90.4%) | HTTP probe, NOT verified browser playback | +| Historical results not classified alive | 114 (9.6%) | Includes blocks, no proof these are broken | +| Official manual catalog | 26 | scripts/official_stations.json | +| Arabic manual catalog | 106 | scripts/arabic_stations.json | +| Manual catalogs with missing language | 0 | 26 + 106 entries | +| Manual catalog duplicate URLs (within each file) | 0 | Case-insensitive literal URL comparison | +| Manual catalog HTTP-only URLs | 0 | 26 + 106 entries | +| Manual catalogs without explicit logo field | 132 | UI fallback exists | +| Global verified playable stations | NOT MEASURED | Historic alive is not audio proof | +| Global country/language/genre completeness | NOT MEASURED | Requires full catalog aggregation | +| Global duplicate broadcaster identities | NOT MEASURED | Same name does not mean same broadcast | +| Before/after real playback success | NOT MEASURED | Needs browser trials | + +## Existing station source registry + +- Official broadcaster endpoints in the hand-curated official registry. +- Arabic curated stations and IPRD Arab-country country playlists. +- Radio Browser API with five server mirrors, pagination, source failover. +- IPRD country playlists globally (excluding forbidden country). +- Community playlists by junguler (unstructured stream discovery). +- Broadcaster portal scraper with seed inventory; Maspero official video pages + use Dailymotion and cannot be interpreted as direct audio. +- eyalev/verified-radio-streams community directory. +- Icecast YP public index. +- Live Radio Browser search (client side; separate from daily snapshot). + +These are existing providers. This PR does not add an unvalidated provider. + +## Root causes found + +- Historic HTTP-based checks can classify 200 and certain access errors + as alive without actual sound bytes. +- 1,185 historic checks are just a small curated/head subset. +- The old absolute minimum of 10,000 catalog records would still permit a + severe incomplete rebuild from 60k to 40k. +- Station identity is URL-based; collapsing records by broadcaster name would + break favorites, history and deep links, and could merge unrelated stations. +- Regional/datacenter IP blocking requires uncertain statuses and repeated + observations; one failed network check is not evidence of a broken stream. +- Official/Arabic curated stations have no explicit logo field, but UI + supplies flags and fallback artwork. + +## Improvements in this PR + +- Safe sampled stream-evidence verification, maximum 48 distinct URLs per + station refresh with curated/failure priorities plus rotating world discovery. +- Bounded 2 KiB sample, 6-second timeout, safe DNS and redirects, one playlist hop. +- Signatures for MP3, AAC, OGG, WAV, FLAC, audio MP4; HLS manifest structural + checks alone are labelled DEGRADED (not verified browser playback). +- Separate 0-100 score and five internal statuses: VERIFIED, DEGRADED, + UNVERIFIED, TEMPORARILY_UNAVAILABLE, BROKEN. +- No delete-on-failure; 404/410 needs three consecutive checked failures to + reach BROKEN; historical last success preserved. +- Observational sample stored separately in data/stream_quality.json. +- Detect major incomplete source crawls: a new catalog over 20% smaller than + previous is quarantined before writing the new stations.json/head/deltas. +- Old URL-based station IDs and player schema remain unchanged. + +## Limits and follow-up + +- Sampling is incremental, not a random population sample. Do not generalize + sample success percentages to 60,732 stations. +- Existing stream_checks.json alive flag is unchanged and NOT the VERIFIED flag. +- HLS segment decode and real mobile/browser playback remain to test. +- URL DNS validation with stdlib has a DNS rebinding TOCTOU limitation. +- GitHub Actions CI and deployment must pass before release is called complete. +- Before/after country and language coverage must be computed from full + catalog during a separate audit job; do not invent coverage figures. +- No alternative stream is accepted without proof it belongs to the same + broadcaster and independent playback evidence. From 88ef24f9d2cf3acdb6bec8421537bdfb773fba7c Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:57:04 +0200 Subject: [PATCH 05/19] docs(data): synchronize roadmap and readme with source-health safeguards --- ROADMAP.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ROADMAP.md b/ROADMAP.md index 0f518ad1..69f44fee 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,6 +6,8 @@ --- +> **صيانة المصادر العالمية — 2026-11-19:** خط الأساس من manifest v34 هو 60,732 محطة؛ آخر سجل قديم فحص 1,185 رابطًا وصنّف 1,071 alive (مش ضمان تشغيل صوت). أضفنا فاحص دليل الصوت التدريجي (48 رابطًا في دورة المحطات)، وحماية تمنع فقد أكثر من 20% من المحطات بسبب جلب ناقص. لا حذف للمحطات ولا تغيير لمعرّفاتها. التفاصيل والتقييم في [تقرير تدقيق المصادر](docs/RADIO_SOURCES_AUDIT_2026-11-19.md). اكتمال التفعيل معلق على CI والنشر ودورة صيانة فعلية. + ## 1) الوضع الحالي — نوفمبر 2026 ### ✅ منجز (سجل مختصر — السجل الكامل في PRs المستودع) From 4659077f67ec2806ea745d8c6b0e3b5b8171690b Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:57:07 +0200 Subject: [PATCH 06/19] docs(data): synchronize roadmap and readme with source-health safeguards --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 96d09219..5a85d990 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ ### 📻 محطات البث +- فحص جودة جديد تدريجي: data/stream_quality.json (48 رابطًا في دورة تحديث المحطات)، يفرق بين دليل صوت حقيقي وبين رد HTTP ناجح فقط أو ملف HLS. فحوص stream_checks.json القديمة ليست تأكيد تشغيل بالمتصفح. تُعزل اللقطات الناقصة جدًا من دون حذف محطات. [تقرير تدقيق المصادر](docs/RADIO_SOURCES_AUDIT_2026-11-19.md). + + - كتالوج كامل من Radio-Browser (كل المحطات غير المكسورة (أكثر من 50,000)، ترتيب: سمعية/عربية/IPRD + الكتالوج الكامل) + في البحث المباشر في FM tab أيضا - بحث فوري في الكتالوج الكامل (~13MB) بعد التحميل الأول — مع ملف إقلاع خفيف stations_head (~530KB) يعرض أول 2500 محطة فورًا، وزياراتك التالية تُحدَّث تفاضليًا بكيلوبايتات عبر IndexedDB - فلترة حسب الدولة/اللغة/الفئة (FACET_OPTIONS + filterFacetOptions(kind) — يحتفظ بالخيار المحدد أثناء التصفية) From e0a8426f4a4b685ee8c682f7dd4e8f1f30e173a3 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:58:02 +0200 Subject: [PATCH 07/19] feat(data): add read-only worldwide coverage and integrity audit --- scripts/catalog_audit.py | 114 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 scripts/catalog_audit.py diff --git a/scripts/catalog_audit.py b/scripts/catalog_audit.py new file mode 100644 index 00000000..99addf00 --- /dev/null +++ b/scripts/catalog_audit.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Offline, read-only catalog audit. No external requests or station deletion.""" +import collections +import json +import re +import sys +import unicodedata +import urllib.parse +from pathlib import Path + + +def url_key(url): + try: + parts = urllib.parse.urlsplit(str(url or "").strip()) + if parts.scheme.lower() not in ("http", "https") or not parts.hostname: + return "" + if parts.username or parts.password: + return "" + port = parts.port + host = parts.hostname.lower() + if port and port not in (80 if parts.scheme == "http" else 443,): + host += ":" + str(port) + return urllib.parse.urlunsplit((parts.scheme.lower(), host, parts.path or "/", + parts.query, "")) + except (TypeError, ValueError): + return "" + + +def normalize_name(name): + text = unicodedata.normalize("NFKC", str(name or "")).casefold() + return re.sub(r"\s+", " ", text).strip() + + +def audit_catalog(stations): + """Counts metadata completeness, URL duplicates and POSSIBLE name collisions. + + Different streams with the same name are NOT merged: country/name is only + a manual review candidate, not evidence of a duplicate station. + """ + countries = collections.Counter() + languages = collections.Counter() + tags = collections.Counter() + codec = collections.Counter() + names = collections.defaultdict(set) + known_urls = set() + duplicate_urls = 0 + missing_country = missing_language = missing_logo = missing_name = http_only = 0 + invalid_url = 0 + for station in stations: + if not isinstance(station, dict): + continue + cc = str(station.get("cc") or "").strip().upper() + country = cc if len(cc) == 2 else "(missing)" + countries[country] += 1 + if country == "(missing)": + missing_country += 1 + langs = [part.strip().casefold() for part in + re.split(r"[,;|]", str(station.get("language") or "")) if part.strip()] + if not langs: + missing_language += 1 + languages.update(set(langs)) + tags.update(set(t.strip().casefold() for t in + str(station.get("tags") or "").split(",") if t.strip())) + codec[str(station.get("codec") or "").strip().upper() or "(missing)"] += 1 + if not station.get("favicon") and not station.get("logo"): + missing_logo += 1 + name = normalize_name(station.get("name")) + if not name or name == "—": + missing_name += 1 + key = url_key(station.get("url")) + if not key: + invalid_url += 1 + elif key in known_urls: + duplicate_urls += 1 + known_urls.add(key) + if str(station.get("url") or "").lower().startswith("http://"): + http_only += 1 + if name and name != "—" and key: + names[(country, name)].add(key) + potential_same_names = sum(1 for urls in names.values() if len(urls) > 1) + result = { + "total": len(stations), + "countries": len([c for c in countries if c != "(missing)"]), + "languages": len(languages), + "missing_country": missing_country, + "missing_language": missing_language, + "missing_logo_field": missing_logo, + "missing_name": missing_name, + "http_only": http_only, + "invalid_url": invalid_url, + "duplicate_stream_urls": duplicate_urls, + "potential_same_country_name_groups_not_merged": potential_same_names, + "country_counts": dict(countries.most_common()), + "language_counts": dict(languages.most_common()), + "codec_counts": dict(codec.most_common()), + "top_tags": dict(tags.most_common(40)), + } + return result + + +def main(): + source = Path(sys.argv[1]) if len(sys.argv) > 1 else ( + Path(__file__).resolve().parents[1] / "data" / "stations.json") + with source.open(encoding="utf-8") as handle: + payload = json.load(handle) + stations = payload.get("stations") if isinstance(payload, dict) else payload + if not isinstance(stations, list): + raise SystemExit("invalid catalog shape") + result = audit_catalog(stations) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() From be0f7dafa265461b66f0ee92ea28f6d0e70c9449 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:58:12 +0200 Subject: [PATCH 08/19] fix(data): use atomic JSON snapshot replacement to keep last good files --- scripts/update_data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/update_data.py b/scripts/update_data.py index 116f9205..91ea95a3 100644 --- a/scripts/update_data.py +++ b/scripts/update_data.py @@ -1912,7 +1912,16 @@ def write_error_report(now_iso): def write_json(path, payload): - path.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") + """Stage JSON beside its target, then atomically replace the last good file.""" + serialized = json.dumps(payload, ensure_ascii=False, indent=1) + target = Path(path) + temporary = target.with_name(target.name + ".tmp") + try: + temporary.write_text(serialized, encoding="utf-8") + temporary.replace(target) + finally: + if temporary.exists(): + temporary.unlink() def write_stream_checks(stations, now_iso): From 576f01eeff2431223e908168195e4ed672e91df7 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:58:15 +0200 Subject: [PATCH 09/19] fix(data): preserve stream-health last known good report atomically --- scripts/stream_quality.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/stream_quality.py b/scripts/stream_quality.py index 3c3c916e..3a562a9c 100644 --- a/scripts/stream_quality.py +++ b/scripts/stream_quality.py @@ -273,6 +273,12 @@ def update_quality_report(stations, path, now_iso, limit=CHECKS_PER_RUN, "checked_this_run": len(picked), "ever_checked": len(results), "not_checked": max(0, len(stations) - len(results)), "status_counts": counts, "results": results} - path.write_text(json.dumps(report, ensure_ascii=False, indent=1), - encoding="utf-8") + temporary = path.with_name(path.name + ".tmp") + try: + temporary.write_text(json.dumps(report, ensure_ascii=False, indent=1), + encoding="utf-8") + temporary.replace(path) + finally: + if temporary.exists(): + temporary.unlink() return report From dc0beace23a7516c182a805855ddef18db4916ee Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:58:24 +0200 Subject: [PATCH 10/19] test(data): measure entire global catalog and atomic rollback in CI --- tests/test_stream_quality.py | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/test_stream_quality.py b/tests/test_stream_quality.py index 52e06eea..5c067b1a 100644 --- a/tests/test_stream_quality.py +++ b/tests/test_stream_quality.py @@ -221,5 +221,52 @@ def fake_probe(url, previous, official, now_iso): self.assertEqual(json.dumps(stations), immutable) +class CatalogCoverageAuditTests(unittest.TestCase): + def test_read_only_full_catalog_baseline_and_invariants(self): + audit = load_module("radio_catalog_audit", "scripts/catalog_audit.py") + path = ROOT / "data" / "stations.json" + with path.open(encoding="utf-8") as handle: + payload = json.load(handle) + rows = payload["stations"] + report = audit.audit_catalog(rows) + self.assertEqual(report["total"], payload["count"]) + self.assertGreater(report["total"], 10000) + self.assertEqual(report["country_counts"].get("IL", 0), 0) + self.assertEqual(sum(report["country_counts"].values()), report["total"]) + # Print grounded live artifact metrics to the CI logs. + compact = {key: report[key] for key in ( + "total", "countries", "languages", "missing_country", + "missing_language", "missing_logo_field", "missing_name", + "http_only", "invalid_url", "duplicate_stream_urls", + "potential_same_country_name_groups_not_merged")} + compact["top_countries"] = dict(list(report["country_counts"].items())[:20]) + compact["top_languages"] = dict(list(report["language_counts"].items())[:20]) + print("GLOBAL RADIO BASELINE:", json.dumps(compact, ensure_ascii=False)) + + def test_same_name_different_stream_is_only_review_candidate(self): + audit = load_module("radio_catalog_audit_names", "scripts/catalog_audit.py") + rows = [ + {"name": "Radio X", "cc": "EG", + "url": "https://radio.example.test/live1", "language": "arabic"}, + {"name": "RADIO X", "cc": "EG", + "url": "https://radio.example.test/live2", "language": "arabic"}, + ] + report = audit.audit_catalog(rows) + self.assertEqual(report["duplicate_stream_urls"], 0) + self.assertEqual(report["potential_same_country_name_groups_not_merged"], 1) + self.assertEqual(report["total"], 2) + + def test_atomic_write_keeps_old_snapshot_when_replace_fails(self): + from unittest import mock + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "stations.json" + path.write_text('{"count": 123}', encoding="utf-8") + with mock.patch.object(Path, "replace", side_effect=OSError("disk error")): + with self.assertRaises(OSError): + crawler.write_json(path, {"count": 456}) + self.assertEqual(path.read_text(encoding="utf-8"), '{"count": 123}') + self.assertFalse((Path(folder) / "stations.json.tmp").exists()) + + if __name__ == "__main__": unittest.main() From b7ec928b270b5af22de2169b421ee7e097870730 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:59:28 +0200 Subject: [PATCH 11/19] fix(data): keep fallback catalog, head and delta manifest in sync --- scripts/update_data.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/scripts/update_data.py b/scripts/update_data.py index 91ea95a3..d4b77ab3 100644 --- a/scripts/update_data.py +++ b/scripts/update_data.py @@ -2617,6 +2617,28 @@ def guard_catalog_snapshot(stations, previous_rows): raise RuntimeError("invalid or duplicated stream URLs in new snapshot") +def publish_station_catalog(payload, previous_rows, now_iso): + """Publish one consistent catalog/head/delta/manifest set for either path. + + Correct pre-existing discrepancies between embedded catalog count and + actual station-list length. Generate deltas from the original snapshot. + """ + stations = payload["stations"] + guard_catalog_snapshot(stations, previous_rows) + updated = payload.get("updated") or now_iso + source = payload.get("source") or "" + normalized = dict(payload, count=len(stations)) + head = stations[:2500] + # Files are committed/deployed together by the serialized Actions job; + # a failed job never publishes the incomplete staging directory. + write_station_deltas(stations, previous_rows, now_iso) + write_json(DATA_DIR / "stations_head.json", { + "updated": updated, "source": source, + "count": len(head), "stations": head, + }) + write_json(DATA_DIR / "stations.json", normalized) + + def write_quality_report(stations, now_iso): """Optional observational health checks; failures never block the catalog.""" try: @@ -2856,26 +2878,16 @@ def main(): f"scraper {len(discovered)} + dailymotion {len(dm_entries)} + " f"radio-browser {len(rb_stations)} + world {len(world_extra)})" ) - write_json( - DATA_DIR / "stations.json", + publish_station_catalog( { "updated": now_iso, "source": "Radio-Browser + IPRD + verified-streams + Icecast YP + community playlists + own scraper + Dailymotion (multi-source)", "count": len(stations), "stations": stations, }, + previous_rows, + now_iso, ) - head = stations[:2500] - write_json( - DATA_DIR / "stations_head.json", - { - "updated": now_iso, - "source": "Radio-Browser + IPRD + verified-streams + Icecast YP + community playlists + own scraper + Dailymotion (multi-source)", - "count": len(head), - "stations": head, - }, - ) - write_station_deltas(stations, previous_rows, now_iso) ok = True write_stream_checks( officials + arabic + arab_extra + stations[:CHECK_HEAD_MAX], @@ -2897,7 +2909,8 @@ def main(): ) merged = merge_curated_into_snapshot(curated, DATA_DIR / "stations.json") if merged: - write_json(DATA_DIR / "stations.json", merged) + previous_rows = read_catalog_rows(DATA_DIR / "stations.json") + publish_station_catalog(merged, previous_rows, now_iso) print(f"stations: merged curated data into existing snapshot ({merged['count']})") ok = True else: From f6bbf8a508f71bda44a51b2d683d63a70efb92e3 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:59:37 +0200 Subject: [PATCH 12/19] test(data): acknowledge existing count drift and verify fallback snapshot synchronization --- tests/test_stream_quality.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_stream_quality.py b/tests/test_stream_quality.py index 5c067b1a..e1c7137d 100644 --- a/tests/test_stream_quality.py +++ b/tests/test_stream_quality.py @@ -229,7 +229,9 @@ def test_read_only_full_catalog_baseline_and_invariants(self): payload = json.load(handle) rows = payload["stations"] report = audit.audit_catalog(rows) - self.assertEqual(report["total"], payload["count"]) + self.assertEqual(report["total"], len(rows)) + # Pre-existing count drift is itself an audited finding. This test + # observes it rather than silently redefining the baseline. self.assertGreater(report["total"], 10000) self.assertEqual(report["country_counts"].get("IL", 0), 0) self.assertEqual(sum(report["country_counts"].values()), report["total"]) @@ -243,6 +245,29 @@ def test_read_only_full_catalog_baseline_and_invariants(self): compact["top_languages"] = dict(list(report["language_counts"].items())[:20]) print("GLOBAL RADIO BASELINE:", json.dumps(compact, ensure_ascii=False)) + def test_fallback_publishes_synced_counts_head_and_delta(self): + from unittest import mock + stations = [{"name": "Radio " + str(i), + "url": "https://station.example.test/" + str(i), "cc": "EG"} + for i in range(4)] + with tempfile.TemporaryDirectory() as folder: + with mock.patch.object(crawler, "DATA_DIR", Path(folder)): + with mock.patch.object(crawler, "guard_catalog_snapshot"): + crawler.publish_station_catalog( + {"updated": "2026-11-19T00:00:00Z", + "source": "fallback snapshot", "count": 1, + "stations": stations}, + [], "2026-11-19T17:35:19Z" + ) + full = json.loads((Path(folder) / "stations.json").read_text()) + head = json.loads((Path(folder) / "stations_head.json").read_text()) + manifest = json.loads((Path(folder) / "stations.manifest.json").read_text()) + delta = json.loads((Path(folder) / "stations_delta/1.json").read_text()) + self.assertEqual(full["count"], 4) + self.assertEqual(head["count"], 4) + self.assertEqual(manifest["count"], 4) + self.assertEqual(len(delta["added"]), 4) + def test_same_name_different_stream_is_only_review_candidate(self): audit = load_module("radio_catalog_audit_names", "scripts/catalog_audit.py") rows = [ From caa3c2ad8321f5ec6b9125f50e5b32c7f03baebf Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:59:48 +0200 Subject: [PATCH 13/19] docs(data): disclose real snapshot count discrepancy from full CI audit --- docs/RADIO_SOURCES_AUDIT_2026-11-19.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md index 2d182533..07ac79a4 100644 --- a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md +++ b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md @@ -4,7 +4,8 @@ Baseline from repository tracked assets at 17:35:19 UTC. | Metric | Before | Meaning | | --- | ---: | --- | -| Catalog entries | 60,732 | data/stations.manifest.json, version 34 | +| Catalog declared count | 60,732 | data/stations.manifest.json v34 AND stations.json count field | +| Actual station rows in stations.json | 60,741 | full JSON array parsed in CI 2026-11-19; mismatch of +9 rows, existing before PR | | Historical stream checks | 1,185 | data/stream_checks.json | | Historical results classified alive | 1,071 (90.4%) | HTTP probe, NOT verified browser playback | | Historical results not classified alive | 114 (9.6%) | Includes blocks, no proof these are broken | From 56c7880c74ff351a349fb60a7499caa15e91bf8b Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:59:51 +0200 Subject: [PATCH 14/19] docs(data): disclose real snapshot count discrepancy from full CI audit --- docs/RADIO_SOURCES_AUDIT_2026-11-19.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md index 07ac79a4..e64b653c 100644 --- a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md +++ b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md @@ -40,6 +40,7 @@ These are existing providers. This PR does not add an unvalidated provider. - Historic HTTP-based checks can classify 200 and certain access errors as alive without actual sound bytes. - 1,185 historic checks are just a small curated/head subset. +- The stored count fields differ from the actual stations array by 9 rows (60,732 vs 60,741) at this baseline. Both normal refresh and fallback now use one snapshot publication function that recalculates count and synchronizes head and delta manifest; the persisted historical data remains unchanged until the next successful station refresh. - The old absolute minimum of 10,000 catalog records would still permit a severe incomplete rebuild from 60k to 40k. - Station identity is URL-based; collapsing records by broadcaster name would From a723eb1193f6e1583d05fa2ee69f35c4d57e1b9e Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 19:59:55 +0200 Subject: [PATCH 15/19] docs(data): disclose real snapshot count discrepancy from full CI audit --- ROADMAP.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 69f44fee..87fd5e20 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -6,7 +6,7 @@ --- -> **صيانة المصادر العالمية — 2026-11-19:** خط الأساس من manifest v34 هو 60,732 محطة؛ آخر سجل قديم فحص 1,185 رابطًا وصنّف 1,071 alive (مش ضمان تشغيل صوت). أضفنا فاحص دليل الصوت التدريجي (48 رابطًا في دورة المحطات)، وحماية تمنع فقد أكثر من 20% من المحطات بسبب جلب ناقص. لا حذف للمحطات ولا تغيير لمعرّفاتها. التفاصيل والتقييم في [تقرير تدقيق المصادر](docs/RADIO_SOURCES_AUDIT_2026-11-19.md). اكتمال التفعيل معلق على CI والنشر ودورة صيانة فعلية. +> **صيانة المصادر العالمية — 2026-11-19:** خط الأساس المعلن في manifest v34 هو 60,732 محطة، بينما صفوف stations.json الفعلية 60,741 (انحراف سابق +9)؛ آخر سجل قديم فحص 1,185 رابطًا وصنّف 1,071 alive (مش ضمان تشغيل صوت). أضفنا فاحص دليل الصوت التدريجي (48 رابطًا في دورة المحطات)، وحماية تمنع فقد أكثر من 20% من المحطات بسبب جلب ناقص. لا حذف للمحطات ولا تغيير لمعرّفاتها. التفاصيل والتقييم في [تقرير تدقيق المصادر](docs/RADIO_SOURCES_AUDIT_2026-11-19.md). اكتمال التفعيل معلق على CI والنشر ودورة صيانة فعلية. ## 1) الوضع الحالي — نوفمبر 2026 From c96a6eb3b4e2f3a98de0600c3e011aa77481ece2 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 20:00:16 +0200 Subject: [PATCH 16/19] feat(data): back off failed stream probes exponentially across daily runs --- scripts/stream_quality.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/scripts/stream_quality.py b/scripts/stream_quality.py index 3a562a9c..ea793983 100644 --- a/scripts/stream_quality.py +++ b/scripts/stream_quality.py @@ -199,6 +199,20 @@ def probe_stream(url, previous=None, official=False, timeout=6, opener=None, return res +def retry_due(previous, now_iso): + """Exponential day backoff: 1, 2, 4, then at most 7 days after failures.""" + failures = int(previous.get("consecutive_failures") or 0) + last = str(previous.get("last_checked") or "")[:10] + if not failures or not last: + return True + try: + elapsed = (datetime.fromisoformat(now_iso[:10]).date() + - datetime.fromisoformat(last).date()).days + except ValueError: + return True + return elapsed >= min(7, 2 ** min(3, failures - 1)) + + def select_incremental(stations, previous, now_iso, limit=CHECKS_PER_RUN): """Prioritize curated, previous failures, then rotate across whole world.""" urls, seen = [], set() @@ -223,15 +237,17 @@ def add(station): if len(chosen) >= min(16, limit // 3): break last = previous.get(row["url"], {}).get("last_checked", "")[:10] - if not last or (datetime.fromisoformat(now_iso[:10]).date() - - datetime.fromisoformat(last).date()).days >= 3: + if retry_due(previous.get(row["url"], {}), now_iso) and ( + not last or (datetime.fromisoformat(now_iso[:10]).date() + - datetime.fromisoformat(last).date()).days >= 3 + ): add(row) for row in urls: if len(chosen) >= min(24, limit // 2): break info = previous.get(row["url"], {}) if info.get("status") in ("TEMPORARILY_UNAVAILABLE", "BROKEN") and ( - info.get("last_checked", "")[:10] != now_iso[:10] + retry_due(info, now_iso) ): add(row) if urls: @@ -239,7 +255,8 @@ def add(station): start = (day * 97 * limit) % len(urls) for i in range(len(urls)): row = urls[(start + i) % len(urls)] - if previous.get(row["url"], {}).get("last_checked", "")[:10] != now_iso[:10]: + info = previous.get(row["url"], {}) + if info.get("last_checked", "")[:10] != now_iso[:10] and retry_due(info, now_iso): add(row) if len(chosen) == limit: break From 19d189579ef8b53d2d0ab7169f83e034d57996f5 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 20:00:23 +0200 Subject: [PATCH 17/19] test(data): check exponential probe backoff across daily refreshes --- tests/test_stream_quality.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_stream_quality.py b/tests/test_stream_quality.py index e1c7137d..f9a601b4 100644 --- a/tests/test_stream_quality.py +++ b/tests/test_stream_quality.py @@ -201,6 +201,20 @@ def test_incremental_sample_is_unique_and_excludes_IL(self): self.assertEqual(len({x["url"] for x in selected}), 12) self.assertTrue(all(x["cc"] != "IL" for x in selected)) + def test_failed_probe_exponential_daily_backoff(self): + date = "2026-11-19T00:00:00+00:00" + for failures, last_checked, due in ( + (1, "2026-11-18T00:00:00+00:00", True), + (2, "2026-11-18T00:00:00+00:00", False), + (2, "2026-11-17T00:00:00+00:00", True), + (3, "2026-11-16T00:00:00+00:00", False), + (3, "2026-11-15T00:00:00+00:00", True), + ): + self.assertEqual(health.retry_due({ + "consecutive_failures": failures, + "last_checked": last_checked, + }, date), due) + def test_quality_report_records_only_sample_not_false_global_success(self): stations = [{"url": "https://radio.example.test/" + str(i), "cc": "EG"} for i in range(30)] From d8ba135096167ba16bab096b89fb5272caccad79 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 20:00:41 +0200 Subject: [PATCH 18/19] docs(data): capture retry backoff and synchronized snapshot publication --- docs/RADIO_SOURCES_AUDIT_2026-11-19.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md index e64b653c..898403aa 100644 --- a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md +++ b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md @@ -59,6 +59,7 @@ These are existing providers. This PR does not add an unvalidated provider. checks alone are labelled DEGRADED (not verified browser playback). - Separate 0-100 score and five internal statuses: VERIFIED, DEGRADED, UNVERIFIED, TEMPORARILY_UNAVAILABLE, BROKEN. +- Exponential retry backoff for failed streams (1, 2, 4, max 7 days). - No delete-on-failure; 404/410 needs three consecutive checked failures to reach BROKEN; historical last success preserved. - Observational sample stored separately in data/stream_quality.json. From bde96edce6a5adfd652ed5652c92924f43467970 Mon Sep 17 00:00:00 2001 From: Ossama Hashim Date: Thu, 19 Nov 2026 20:00:43 +0200 Subject: [PATCH 19/19] docs(data): capture retry backoff and synchronized snapshot publication --- docs/RADIO_SOURCES_AUDIT_2026-11-19.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md index 898403aa..8ea82358 100644 --- a/docs/RADIO_SOURCES_AUDIT_2026-11-19.md +++ b/docs/RADIO_SOURCES_AUDIT_2026-11-19.md @@ -63,6 +63,7 @@ These are existing providers. This PR does not add an unvalidated provider. - No delete-on-failure; 404/410 needs three consecutive checked failures to reach BROKEN; historical last success preserved. - Observational sample stored separately in data/stream_quality.json. +- Atomically stage individual JSON files; normal and fallback station refreshes synchronize station, head, delta and manifest counts. - Detect major incomplete source crawls: a new catalog over 20% smaller than previous is quarantined before writing the new stations.json/head/deltas. - Old URL-based station IDs and player schema remain unchanged.