diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f0baf62..4c6aec72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,18 +246,17 @@ jobs: - name: Operational-readiness gate run: python tools/validate_operational_readiness.py apps/web/data/latest-state.json - # ADR 0005 round-trip: the Rust CLI reads a real observation report, corrects the - # activity forecast through the Kalman primitive, and the emitted Assimilation-mode - # snapshot must pass the shared Python validators — Rust reading, Rust writing, - # Python judging, one contract. + # Use deterministic synthetic reports from the actual Python producer: + # fresh activity must assimilate, while missing/stale activity must retain + # the prior. The app's historical report is not guaranteed to admit analysis. + # Both modes must pass shared schema/readiness validators after Rust transport. - name: Assimilation round-trip (Rust -> shared validators) shell: bash + env: + SOL_REQUIRE_CLI: "1" run: | - cargo run --quiet --locked -p solar-cli -- simulate --steps 48 --dt-hours 1 --seed 42 \ - --observations apps/web/data/latest-observations.json --out /tmp/assimilated.json - grep -q '"mode": "Assimilation"' /tmp/assimilated.json - python tools/validate_snapshot.py /tmp/assimilated.json - python tools/validate_operational_readiness.py /tmp/assimilated.json + cargo build --quiet --locked -p solar-cli + PYTHONPATH=tools python -m unittest discover -s tests/python -p test_activity_observation.py -v - name: Regenerate deterministic solar-cycle series run: python tools/generate_series.py diff --git a/BUILD_NOTES.md b/BUILD_NOTES.md index 3c8f9062..70b9b3d2 100644 --- a/BUILD_NOTES.md +++ b/BUILD_NOTES.md @@ -66,47 +66,53 @@ Patched after reviewing NWS Service Change Notice 26-21: - Extended cached snapshots with `observed-context.v1` so SWPC solar regions, sunspot rows, GOES flare rows, and cycle indices can tune the research fixture while preserving provenance and normalized-unit caveats. - Updated the web UI to show daily feed status separately from snapshot readiness. -## Publishing `solar-ephemeris` to crates.io - -> **Status:** `solar-ephemeris` is live on crates.io — -> . **0.1.1** was the first publish -> (2026-07-20); **0.2.0** (2026-07-20) is current and carries the 108-star -> bright-star catalogue. The steps below drive subsequent version bumps; the -> guard skips any version already on crates.io, so a scheduled run is a no-op -> until the workspace version changes. - -The `solar-ephemeris` crate is released to crates.io by the -`.github/workflows/publish-crate.yml` workflow. It runs on a weekly schedule -(Mondays 07:30 UTC) and on demand via **workflow_dispatch**. A guard step queries -crates.io and skips the publish when the current workspace version is already -live, so scheduled runs stay green between releases — publishing is driven by -version bumps, not by the clock. - -### One-time setup: the `CARGO_REGISTRY_TOKEN` secret - -The publish step authenticates with a crates.io API token stored in the repository -secret `CARGO_REGISTRY_TOKEN`. Without it the workflow still runs, but the publish -step fails to authenticate. - -1. On crates.io, go to **Account Settings -> API Tokens -> New Token**. - - Scopes: `publish-new` (needed for the first-ever publish of the crate name) - and `publish-update` (later version bumps). - - Crate scope: restrict the token to `solar-ephemeris` (least privilege). - - Optionally set an expiry, and copy the token (shown only once). -2. Add it to the repo at **Settings -> Secrets and variables -> Actions -> - New repository secret**: name it exactly `CARGO_REGISTRY_TOKEN`, value = the - token. (Or locally: `gh secret set CARGO_REGISTRY_TOKEN --repo Protonmatter/sol`.) - -The token owner must be an owner of the `solar-ephemeris` crate on crates.io once -it is published, or later version-bump publishes return 403. - -### Cutting a release - -1. Bump `version` in the workspace `[workspace.package]` table in the root - `Cargo.toml` (the crate inherits it via `version.workspace = true`), and update - `Cargo.lock`. -2. Land it on `master` through the normal CI-gated PR flow. -3. Either wait for the next weekly run or trigger it now: **Actions -> "Publish - solar-ephemeris to crates.io" -> Run workflow**. The guard detects the new - version is not yet on crates.io and publishes it with `cargo publish -p - solar-ephemeris --locked`. +## Registry history and current package qualification + +Reviewed: 2026-09-28. The historical registry publications remain valid: +[0.1.1](https://docs.rs/solar-ephemeris/0.1.1/solar_ephemeris/) was first published +on 2026-07-20, followed by +[0.2.0](https://docs.rs/solar-ephemeris/0.2.0/solar_ephemeris/) that day with the +108-star bright-star catalogue. The +[crates.io API record](https://crates.io/api/v1/crates/solar-ephemeris) confirms +both versions exist and are not yanked as of this review. The crates.io browser +pages returned HTTP 404 during the review; that does not establish registry absence. +The links above open the versioned API documentation. Published versions and the +current checkout's v3 contract are separate artifacts. + +The current [package workflow](.github/workflows/publish-crate.yml) runs only on +**workflow_dispatch**, with a required exact 40-character `source_sha`. Its retained +name is "Publish solar-ephemeris to crates.io", but its implemented action is +**package qualification with publication held**. There is no weekly schedule, +registry credential input or registry publication step. A version bump does not +authorize publication. + +### Qualify an exact source candidate + +Use a clean checkout of the reviewed commit. The local package checks are: + +```bash +cargo test -p solar-ephemeris --locked +cargo package --list -p solar-ephemeris --locked +cargo publish --dry-run -p solar-ephemeris --locked +``` + +The dry run can access the registry index; it does not upload the crate. Retain its +log and the resulting `.crate` bytes for package-content and notice review. For a +separately authorized hosted qualification, supply the reviewed commit explicitly: + +```bash +gh workflow run publish-crate.yml --repo Protonmatter/sol -f source_sha=REVIEWED_40_CHARACTER_COMMIT_SHA +``` + +The workflow checks out that SHA, tests/packages it, records exact-source package +evidence with `tools/release_crate.py`, and uploads +`qualified-crate-candidate-RUN_ID-RUN_ATTEMPT` with 90-day retention. After successful +qualification/upload, its final hold step deliberately exits **1** to make the +publication boundary visible. Earlier failures may prevent candidate creation; +inspect the exact run's steps and artifacts before claiming qualification. + +Actual publication remains held pending exact-source CI, accepted package +digest/contents/notices and separately protected registry authority. Do not add a +registry secret to bypass this hold. Any future publication path also needs +post-publication checksum verification, because publishing can repackage source. +See [release delivery](docs/RELEASE_DELIVERY.md) for the governing evidence boundaries. diff --git a/CHANGELOG.md b/CHANGELOG.md index 13093f45..74cc9507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -244,7 +244,7 @@ Review round (all findings from the Codex PR reviewer, each verified before acti ### Published -- First [crates.io](https://crates.io/crates/solar-ephemeris) release of the +- First crates.io release ([versioned API documentation](https://docs.rs/solar-ephemeris/0.1.1/solar_ephemeris/)) of the **`solar-ephemeris`** crate — a zero-dependency VSOP2013 + ELP-MPP02 + TOP2013 ephemeris and topocentric sky engine, validated against JPL Horizons to arcsecond class. Install with `cargo add solar-ephemeris`. @@ -253,7 +253,9 @@ Review round (all findings from the Codex PR reviewer, each verified before acti - Add `.github/workflows/publish-crate.yml` — scheduled (weekly) and on-demand `cargo publish -p solar-ephemeris`, guarded to skip versions already on crates.io so - scheduled runs stay green between releases. + scheduled runs stay green between releases. This describes the workflow at that + release; the current workflow performs manual exact-source qualification with + publication held, as documented in [build notes](BUILD_NOTES.md). - Harden `.github/workflows/ephemeris-accuracy.yml` — SHA-pin all actions, add a job timeout, and build with `--locked`, matching the repo's other workflows. - Add `.github/workflows/docs.yml` + `tools/validate_docs.py` — offline Markdown @@ -264,5 +266,5 @@ Review round (all findings from the Codex PR reviewer, each verified before acti - Document the crates.io release process in `BUILD_NOTES.md`. [Unreleased]: https://github.com/Protonmatter/sol/compare/master...HEAD -[0.2.0]: https://crates.io/crates/solar-ephemeris/0.2.0 -[0.1.1]: https://crates.io/crates/solar-ephemeris/0.1.1 +[0.2.0]: https://docs.rs/solar-ephemeris/0.2.0/solar_ephemeris/ +[0.1.1]: https://docs.rs/solar-ephemeris/0.1.1/solar_ephemeris/ diff --git a/apps/web/engine.js b/apps/web/engine.js index 55e17ed9..9f2979b1 100644 --- a/apps/web/engine.js +++ b/apps/web/engine.js @@ -40,7 +40,7 @@ export function engineReady() { // Read the bytes immediately and copy them out before any further wasm call. export function simulateSnapshot({ seed = 42, steps = 24, dtHours = 1, activity = 0.9, lon = 72, lat = 36 } = {}) { validateSolarRequest({seed,steps,dtHours,activity,lon,lat}); - if (!wasmExports) throw new Error("engine not loaded"); + if (!engineReady()) throw new Error("engine not loaded"); const ptr = wasmExports.simulate(seed, steps, dtHours, activity, lon, lat); const len = wasmExports.result_len(); const view = new Uint8Array(wasmExports.memory.buffer, ptr, len); diff --git a/apps/web/js/bodyData.js b/apps/web/js/bodyData.js index c607d1aa..474fd8a0 100644 --- a/apps/web/js/bodyData.js +++ b/apps/web/js/bodyData.js @@ -33,7 +33,7 @@ export const AU_KM = 149597870.7; /** @type {Record} */ export const BODY = { Sun: { - radiusKm: 695700, polarKm: 695700, massKg: 1.9885e30, densityGcm3: 1.408, + radiusKm: SUN_RADIUS_KM, polarKm: SUN_RADIUS_KM, massKg: 1.9885e30, densityGcm3: 1.408, gravity: 274.0, escapeKms: 617.5, rotationHours: 609.12 /* 25.38 d Carrington sidereal */, tiltDeg: 7.25, poleRaDeg: 286.13, poleDecDeg: 63.87, poleRaDotDegPerCty: 0.0, poleDecDotDegPerCty: 0.0, w0Deg: 84.176, wDotDegPerDay: 14.1844, magDipoleEarth: 0, magnetosphere: true, diff --git a/apps/web/js/orrery.js b/apps/web/js/orrery.js index 8e5b1dc2..ff611073 100644 --- a/apps/web/js/orrery.js +++ b/apps/web/js/orrery.js @@ -68,7 +68,7 @@ import { galShear, sunGalacticPos, buildGalaxyModel, buildGalObjectList, buildCatalogStarsGalactic, buildNeighbourhoodModel, neighbourhoodPos, } from "./orreryGalaxy.js?v=dcca6290db"; -import { renderDetail, renderMoonDetail, renderSmallDetail, updateLiveDetailFacts, updateDetailAppearance } from "./orreryDetail.js?v=dcca6290de"; +import { renderDetail, renderMoonDetail, renderSmallDetail, updateSmallDetailFacts, updateLiveDetailFacts, updateDetailAppearance } from "./orreryDetail.js?v=dcca6290de"; import { renderStarDetail } from "./starDetail.js?v=dcca6290db"; import { buildEarthMapSliced, buildFeatureMap } from "./surfacemap.js?v=dcca6290db"; import { resolveDisplayRadii, moonGuideVisible } from "./displayGeometry.js?v=dcca6290db"; @@ -1212,13 +1212,14 @@ function buildSmallBuffers() { // Recompute every small-body marker for the current `renderUnix`, and upload them as one point buffer. function rebuildSmallBodies() { - if (!gl) return; // context lost — keep the previous CPU-side list until restore smallBodies = []; const jy2k = timeJy2k(state.renderUnix); // `el` carries the source record so the detail card can show the orbit (a, e, i, period). for (const b of DWARFS) smallBodies.push({ name: b.n, pos: bodyXYZ(b, jy2k), col: b.col, kind: "dwarf", note: b.note, el: b }); for (const c of COMETS) smallBodies.push({ name: c.n, pos: bodyXYZ(c, jy2k), col: c.col, kind: "comet", note: c.note, el: c }); for (const p of PROBES) smallBodies.push({ name: p.n, pos: probeXYZ(p), col: p.col, kind: "probe", note: p.note, el: p }); + updateSmallDetailFacts(smallBodies.find(body=>body.name===state.selected)); + if (!gl) return; // CPU facts still follow explicit time changes during context loss. // The CPU-side list above is ALWAYS built: the Focus dropdown and the detail panel resolve // small bodies through it even while the drawn layer is hidden. Only the GPU markers are // gated on the checkbox (drawSmallBodies/pick/labels each gate themselves). @@ -3269,7 +3270,9 @@ async function enterOrreryInner() { // frame corrupting the Sun surface's layout for the rest of the session. canvas.style.display = ""; try { - const {unix}=validateSystemRequest({unix:effectiveBaseUnix() + state.simElapsed}); + // Leaving suspends the scene clock. Reentry/Retry refreshes its metadata, not + // its accepted time; only first entry and explicit Now/time inputs use wall time. + const {unix}=validateSystemRequest({unix:state.bodies.length===9 ? state.renderUnix : effectiveBaseUnix() + state.simElapsed}); const [,snapshot]=await Promise.all([loadSkyEngine(),requestSystemSnapshot(unix)]); if(!state.active||generation!==systemGeneration)return; state.bodies=snapshot.bodies.map(body=>({...body}));state.renderUnix=unix;state.metadataUnix=unix;state.engineError="";lastFullSnapshot=performance.now(); diff --git a/apps/web/js/orreryDetail.js b/apps/web/js/orreryDetail.js index 586e42a4..ed757b82 100644 --- a/apps/web/js/orreryDetail.js +++ b/apps/web/js/orreryDetail.js @@ -106,10 +106,9 @@ export function renderSmallDetail(s) { const add = (k, v) => { if (v == null) return; const dt = document.createElement("dt"); dt.textContent = k; - const dd = document.createElement("dd"); dd.textContent = v; dl.append(dt, dd); + const dd = document.createElement("dd"); dd.textContent = v; dd.dataset.metric = k; dl.append(dt, dd); }; - const distAU = Math.hypot(s.pos[0], s.pos[1], s.pos[2]); - add("Distance from Sun", `${distAU.toFixed(2)} AU · light ${(distAU * 8.317 / 60).toFixed(1)} h`); + add("Distance from Sun", smallBodyDistanceLabel(s)); const el = s.el || {}; if (s.kind !== "probe" && el.a != null) { add("Semi-major axis", `${el.a.toFixed(2)} AU`); @@ -132,6 +131,23 @@ export function renderSmallDetail(s) { host.appendChild(card); } +function smallBodyDistanceLabel(body) { + const distAU = Math.hypot(...body.pos); + return `${distAU.toFixed(2)} AU · light ${(distAU * 8.317 / 60).toFixed(1)} h`; +} + +// Called when current-epoch markers are rebuilt, including animation frames. +// Updating only the metric retains open source disclosures and keyboard focus. +export function updateSmallDetailFacts(body) { + if (!body) return; + const host = document.getElementById("orreryDetail"); + if (host?.querySelector(".system-detail > strong")?.textContent !== body.name) return; + const value = smallBodyDistanceLabel(body); + for (const node of host.querySelectorAll("[data-metric]")) { + if (node.getAttribute("data-metric") === "Distance from Sun" && node.textContent !== value) node.textContent = value; + } +} + // Render the facts card for `name` into #orreryDetail. `live` is the body's row from the // current system snapshot (distances/speed/phase/magnitude/equilibrium temp), or undefined. export function renderDetail(name, live, appearanceState = {}) { diff --git a/crates/solar-cli/src/main.rs b/crates/solar-cli/src/main.rs index 6824b677..fa46acba 100644 --- a/crates/solar-cli/src/main.rs +++ b/crates/solar-cli/src/main.rs @@ -190,12 +190,15 @@ fn assess_observations( .ok() .map(|assessment| (value, assessment)) }); - let observed_activity = context.map(|(_, assessment)| assessment.activity_index); - // Freshness is judged from the report's own generation-time evaluation (age vs the - // per-feed limits), so the run is reproducible from the file alone: the gain is the - // fraction of feeds that were fresh when the report was written. - let (fresh, total) = context - .map(|(_, assessment)| (assessment.fresh, assessment.total)) + let activity = context.and_then(|(value, assessment)| { + snapshot_validation::assess_activity_observation(value, assessment, &evidence) + }); + let observed_activity = activity.map(|assessment| assessment.activity_index); + // Evaluate the report's recorded ages, never the wall clock. New descriptors + // include only fresh activity contributors; unrelated feed freshness cannot + // admit a missing/stale signal. Historical simple reports retain their gain. + let (fresh, total) = activity + .map(|assessment| (assessment.fresh, assessment.total)) .unwrap_or((0, 0)); let freshness_gain = if total == 0 { 0.0 @@ -213,7 +216,7 @@ fn assess_observations( && report_source_mode.is_some(); if !usable { let reason = if observed_activity.is_none() { - "no valid observed_context activity/freshness values" + "no attributable fresh activity observation; fixture defaults cannot be assimilated" } else if evidence.is_empty() { "no complete observation frames with attributable provenance" } else if report_source_mode.is_none() { diff --git a/crates/solar-cli/src/snapshot_validation.rs b/crates/solar-cli/src/snapshot_validation.rs index a34eef52..e56dcef9 100644 --- a/crates/solar-cli/src/snapshot_validation.rs +++ b/crates/solar-cli/src/snapshot_validation.rs @@ -288,6 +288,117 @@ pub(crate) fn assess_observed_context( }) } +/// New producers distinguish observed activity from the illustrative fixture default. +/// Historical snapshots remain readable; legacy Python reports can be assimilated +/// only when every contributing proxy (rather than unrelated wind/magnetic feeds) +/// has attributable evidence and known fresh age. +pub(crate) fn assess_activity_observation( + context: &JsonValue, + legacy: ObservedContextAssessment, + evidence: &[JsonValue], +) -> Option { + const IDS: [&str; 5] = [ + "swpc-solar-regions", + "swpc-sunspot-report", + "swpc-goes-xray-flares-7-day", + "swpc-f107-cm-flux", + "swpc-observed-cycle-indices", + ]; + let freshness = context.get("signal_freshness")?; + let eligible = |id: &str| { + IDS.contains(&id) + && freshness.get(id).is_some_and(|entry| { + entry.get("stale").and_then(JsonValue::as_bool) == Some(false) + && entry + .get("age_hours") + .and_then(JsonValue::as_f64) + .is_some_and(|age| age.is_finite() && (0.0..=48.0).contains(&age)) + }) + && evidence.iter().any(|frame| { + frame.get("id").and_then(JsonValue::as_str) == Some(id) + && frame + .get("provenance") + .and_then(|p| p.get("active")) + .and_then(JsonValue::as_bool) + != Some(false) + }) + }; + if let Some(observation) = context.get("activity_observation") { + if observation.get("status").and_then(JsonValue::as_str) != Some("available") { + return None; + } + let value = observation.get("value")?.as_f64()?; + let contributors = observation.get("contributors")?.as_array()?; + if !value.is_finite() + || !(0.0..=1.0).contains(&value) + || contributors.is_empty() + || contributors.len() > IDS.len() + { + return None; + } + let mut ids = std::collections::BTreeSet::new(); + let mut sum = 0.0; + for contributor in contributors { + let id = contributor.get("id")?.as_str()?; + let contribution = contributor.get("value")?.as_f64()?; + if !eligible(id) + || !ids.insert(id) + || !contribution.is_finite() + || !(0.0..=1.0).contains(&contribution) + { + return None; + } + sum += contribution; + } + // Producers round the mean to six decimal places; no unbound alternative + // activity value may be substituted into the accepted descriptor. + if (value - sum / contributors.len() as f64).abs() > 0.000_000_51 { + return None; + } + return Some(ObservedContextAssessment { + activity_index: value, + fresh: contributors.len(), + total: contributors.len(), + }); + } + if let Some(proxies) = context.get("activity_proxy_sources") { + let mut contributors = Vec::new(); + for (key, id) in [ + ("solar_region_rows", IDS[0]), + ("sunspot_rows", IDS[1]), + ("goes_xray_flares_7_day_rows", IDS[2]), + ] { + let count = proxies.get(key)?.as_f64()?; + if !count.is_finite() || count < 0.0 || count.fract() != 0.0 { + return None; + } + if count > 0.0 { + contributors.push(id); + } + } + if !matches!(proxies.get("latest_f107")?, JsonValue::Null) { + let flux = proxies.get("latest_f107")?.as_f64()?; + if !flux.is_finite() { + return None; + } + // Legacy metadata cannot bind a mixed daily/monthly fallback safely. + // Require the daily signal and its exact provenance to be fresh. + contributors.push(IDS[3]); + } + if contributors.is_empty() || !contributors.iter().all(|id| eligible(id)) { + return None; + } + return Some(ObservedContextAssessment { + activity_index: legacy.activity_index, + fresh: contributors.len(), + total: contributors.len(), + }); + } + // Existing native F10.7 and historical simple reports retain their explicit + // observation contract. They never carry Python's default/count envelope. + Some(legacy) +} + pub fn validate(raw: &str) -> Result<(), String> { let value = parse_json(raw).map_err(|err| format!("snapshot JSON: {err}"))?; if value.get("schema_version").and_then(JsonValue::as_str) != Some("solar-state-snapshot.v3") { diff --git a/crates/solar-cli/tests/observation_contract.rs b/crates/solar-cli/tests/observation_contract.rs index 53c6d23c..4a3cda7a 100644 --- a/crates/solar-cli/tests/observation_contract.rs +++ b/crates/solar-cli/tests/observation_contract.rs @@ -232,3 +232,159 @@ fn malformed_freshness_entries_cannot_influence_analysis() { ); assert_safe_synthetic(&snapshot, "malformed-freshness"); } + +#[test] +fn activity_availability_and_contributor_freshness_control_analysis() { + let root = FixtureDirectory::new(); + let report = root.0.join("activity-report.json"); + let snapshot = root.0.join("activity-snapshot.json"); + let cases = [ + ( + "unavailable", + r#"{"activity_index":0.9,"signal_freshness":{"rtsw_wind_1m.json":{"stale":false}},"activity_observation":{"status":"unavailable","value":null,"contributors":[]}}"#, + "Synthetic", + 0.9, + ), + ( + "stale-contributor", + r#"{"activity_index":0.9,"signal_freshness":{"rtsw_wind_1m.json":{"stale":false},"swpc-f107-cm-flux":{"stale":true}},"activity_observation":{"status":"available","value":0.25,"contributors":[{"id":"swpc-f107-cm-flux","value":0.25}]}}"#, + "Synthetic", + 0.9, + ), + ( + "available-contributor", + r#"{"activity_index":0.9,"signal_freshness":{"rtsw_wind_1m.json":{"stale":true},"swpc-f107-cm-flux":{"stale":false,"age_hours":0}},"activity_observation":{"status":"available","value":0.25,"contributors":[{"id":"swpc-f107-cm-flux","value":0.25}]}}"#, + "Assimilation", + 0.38, + ), + ( + "legacy-empty-proxies", + r#"{"activity_index":0.9,"signal_freshness":{"rtsw_wind_1m.json":{"stale":false}},"activity_proxy_sources":{"solar_region_rows":0,"sunspot_rows":0,"goes_xray_flares_7_day_rows":0,"latest_f107":null}}"#, + "Synthetic", + 0.9, + ), + ( + "legacy-stale-activity", + r#"{"activity_index":0.25,"signal_freshness":{"rtsw_wind_1m.json":{"stale":false},"swpc-f107-cm-flux":{"stale":true}},"activity_proxy_sources":{"solar_region_rows":0,"sunspot_rows":0,"goes_xray_flares_7_day_rows":0,"latest_f107":65}}"#, + "Synthetic", + 0.9, + ), + ]; + for (id, context, mode, activity) in cases { + fs::write(&report, format!(r#"{{"schema_version":"observation-frame.v1","source_mode":"cached","frames":[{{"id":"swpc-f107-cm-flux","layer_kind":"observed","source_mode":"cached","provenance":{{"source":"synthetic regression","active":true,"raw_source_metadata":{{}}}},"quality_flags":["offline test"]}}],"observed_context":{context}}}"#)).unwrap(); + let result = simulate(&report, &snapshot); + assert!(result.status.success(), "{id}: {}", text(&result.stderr)); + let value = parse_json(&fs::read_to_string(&snapshot).unwrap()).unwrap(); + let run = value.get("run").unwrap(); + assert_eq!( + run.get("mode").and_then(JsonValue::as_str), + Some(mode), + "{id}" + ); + assert_eq!( + run.get("activity_index").and_then(JsonValue::as_f64), + Some(activity), + "{id}" + ); + if mode == "Synthetic" { + assert_eq!( + value + .get("uncertainty") + .unwrap() + .get("activity") + .unwrap() + .get("variance") + .and_then(JsonValue::as_f64), + Some(0.04), + "{id}: missing evidence cannot reduce uncertainty" + ); + } + } +} + +#[test] +fn serialized_activity_stage_matches_shared_literal_cases() { + let root = FixtureDirectory::new(); + let snapshot = root.0.join("stage.json"); + let cases = parse_json(include_str!( + "../../../tests/fixtures/activity-stage-cases.json" + )) + .unwrap(); + for case in cases.as_array().unwrap() { + let activity = case.get("activity").unwrap().as_f64().unwrap().to_string(); + let result = cli(&[ + "simulate", + "--steps", + "0", + "--activity", + &activity, + "--out", + snapshot.to_str().unwrap(), + ]); + assert!(result.status.success(), "{}", text(&result.stderr)); + let output = parse_json(&fs::read_to_string(&snapshot).unwrap()).unwrap(); + assert_eq!( + output.get("learning").unwrap().get("cycle_stage"), + case.get("stage"), + "activity {activity}" + ); + } +} + +#[test] +fn malformed_activity_descriptors_do_not_borrow_other_feed_freshness() { + let root = FixtureDirectory::new(); + let report = root.0.join("malformed-activity.json"); + let snapshot = root.0.join("snapshot.json"); + let cases = [ + ( + "missing-evidence", + "swpc-sunspot-report", + 0.0, + r#"[{"id":"swpc-f107-cm-flux","value":0.25}]"#, + 0.25, + ), + ( + "stale-age-with-false-flag", + "swpc-f107-cm-flux", + 49.0, + r#"[{"id":"swpc-f107-cm-flux","value":0.25}]"#, + 0.25, + ), + ( + "future-age-with-false-flag", + "swpc-f107-cm-flux", + -1.0, + r#"[{"id":"swpc-f107-cm-flux","value":0.25}]"#, + 0.25, + ), + ( + "duplicate-contributor", + "swpc-f107-cm-flux", + 0.0, + r#"[{"id":"swpc-f107-cm-flux","value":0.25},{"id":"swpc-f107-cm-flux","value":0.25}]"#, + 0.25, + ), + ( + "incoherent-mean", + "swpc-f107-cm-flux", + 0.0, + r#"[{"id":"swpc-f107-cm-flux","value":0.25}]"#, + 0.9, + ), + ( + "unrelated-feed-as-contributor", + "rtsw_wind_1m.json", + 0.0, + r#"[{"id":"rtsw_wind_1m.json","value":0.25}]"#, + 0.25, + ), + ("empty-contributors", "swpc-f107-cm-flux", 0.0, "[]", 0.25), + ]; + for (id, frame_id, age, contributors, activity) in cases { + fs::write(&report, format!(r#"{{"schema_version":"observation-frame.v1","source_mode":"cached","frames":[{{"id":"{frame_id}","layer_kind":"observed","source_mode":"cached","provenance":{{"source":"synthetic regression","active":true,"raw_source_metadata":{{}}}},"quality_flags":["offline test"]}}],"observed_context":{{"activity_index":0.25,"signal_freshness":{{"rtsw_wind_1m.json":{{"stale":false,"age_hours":0}},"swpc-f107-cm-flux":{{"stale":false,"age_hours":{age}}}}},"activity_observation":{{"status":"available","value":{activity},"contributors":{contributors}}}}}}}"#)).unwrap(); + let result = simulate(&report, &snapshot); + assert!(result.status.success(), "{id}: {}", text(&result.stderr)); + assert_safe_synthetic(&snapshot, id); + } +} diff --git a/crates/solar-ephemeris/README.md b/crates/solar-ephemeris/README.md index 1bbf768c..9a6761e7 100644 --- a/crates/solar-ephemeris/README.md +++ b/crates/solar-ephemeris/README.md @@ -1,7 +1,7 @@ # solar-ephemeris -[![crates.io](https://img.shields.io/crates/v/solar-ephemeris.svg)](https://crates.io/crates/solar-ephemeris) -[![docs.rs](https://docs.rs/solar-ephemeris/badge.svg)](https://docs.rs/solar-ephemeris) +[![Published crate version](https://img.shields.io/crates/v/solar-ephemeris.svg)](https://docs.rs/solar-ephemeris/0.2.0/solar_ephemeris/) +[![docs.rs](https://docs.rs/solar-ephemeris/badge.svg)](https://docs.rs/solar-ephemeris/0.2.0/solar_ephemeris/) A **zero-dependency** Rust ephemeris and topocentric sky engine: @@ -28,9 +28,13 @@ the current v3 evidence records are explicitly source-theory parity, not indepen solar-ephemeris = { path = "path/to/sol/crates/solar-ephemeris" } ``` -V3 is a local preview; published releases may retain a different contract. Releases are listed on -[crates.io](https://crates.io/crates/solar-ephemeris); rendered API docs on -[docs.rs](https://docs.rs/solar-ephemeris). Zero dependencies — it pulls in nothing else. +The current v3 source and published crate releases are separate artifacts. Versioned +API documentation is available for published +[0.2.0](https://docs.rs/solar-ephemeris/0.2.0/solar_ephemeris/) and +[0.1.1](https://docs.rs/solar-ephemeris/0.1.1/solar_ephemeris/); consult the selected +version's contract before using it. The current repository workflow qualifies exact +source packages but holds further publication; see [build notes](../../BUILD_NOTES.md). +Zero dependencies — it pulls in nothing else. The engine emits a versioned `ephemeris-snapshot.v3` JSON document for a given time and observer. The runnable `snapshot` example shows end-to-end usage: diff --git a/crates/solar-gpu/shaders/assimilate_blend.wgsl b/crates/solar-gpu/shaders/assimilate_blend.wgsl deleted file mode 100644 index 7d0ad9e4..00000000 --- a/crates/solar-gpu/shaders/assimilate_blend.wgsl +++ /dev/null @@ -1,11 +0,0 @@ -// Placeholder WGSL kernel for diagonal Kalman-style field correction. -@group(0) @binding(0) var forecast: array; -@group(0) @binding(1) var observation: array; -@group(0) @binding(2) var forecast_variance: array; -@group(0) @binding(3) var observation_variance: array; -@group(0) @binding(4) var analysis: array; - -@compute @workgroup_size(256) -fn main(@builtin(global_invocation_id) gid: vec3) { - // Implement K = P_f / (P_f + R); x_a = x_f + K(y - x_f). -} diff --git a/crates/solar-gpu/shaders/diffuse.wgsl b/crates/solar-gpu/shaders/diffuse.wgsl deleted file mode 100644 index 32996064..00000000 --- a/crates/solar-gpu/shaders/diffuse.wgsl +++ /dev/null @@ -1,8 +0,0 @@ -// Placeholder WGSL kernel for spherical-grid diffusion. -@group(0) @binding(0) var br_current: array; -@group(0) @binding(1) var br_next: array; - -@compute @workgroup_size(16, 16) -fn main(@builtin(global_invocation_id) gid: vec3) { - // Implement laplacian stencil with polar boundary handling in v0.2. -} diff --git a/crates/solar-gpu/shaders/rotate.wgsl b/crates/solar-gpu/shaders/rotate.wgsl deleted file mode 100644 index 9bf80815..00000000 --- a/crates/solar-gpu/shaders/rotate.wgsl +++ /dev/null @@ -1,9 +0,0 @@ -// Placeholder WGSL kernel for latitude-dependent differential rotation. -// v0.2 task: implement semi-Lagrangian advection over br_current -> br_next. -@group(0) @binding(0) var br_current: array; -@group(0) @binding(1) var br_next: array; - -@compute @workgroup_size(16, 16) -fn main(@builtin(global_invocation_id) gid: vec3) { - // Implement in v0.2 using grid uniforms and interpolation. -} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 603e0f1e..654eebc2 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -40,6 +40,13 @@ rewriting its original raw files; review provenance gaps before use. selection only; failure returns 1 and retains the last derived pointer. Argument errors can return 2. Failure-attempt evidence remains distinct from healthy feed status. +Critical provider JSON must contain eligible timestamped numeric observations; +an HTTP-success error object is rejected. Empty optional event catalogues remain +valid. `--fail-on-degraded` checks acquisition failure/fallback, while observation +age is assessed separately. An illustrative default activity is not an observation +and cannot authorize assimilation or shrink variance. Inspect +`observed_context.activity_observation` for contributor availability in new reports. + Omitting `--skip-fetch` performs network acquisition and requires separate authorization. Do not infer permission from scheduling examples or use a demonstration fixture as a live observation. See the separately maintained [data update playbook](DATA_UPDATE_PLAYBOOK.md) diff --git a/docs/SPEC.md b/docs/SPEC.md index 6d3d3407..aa27327b 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -343,6 +343,13 @@ in illustrative fixture context; all-stale reports do not create a scalar analys Ingestion MUST reject unattributable records before selecting the newest usable row or deriving observed activity. Valid magnetic or wind evidence cannot authorize an activity value derived from an unattributable F10.7 row. +New report producers MUST distinguish unavailable activity observations from the +illustrative fixture default. An explicit activity-observation descriptor binds its +value to named, fresh, attributable activity contributors and matching evidence. +Wind/magnetic freshness MUST NOT authorize a missing activity observation or reduce +its uncertainty. Explicit malformed/unavailable descriptors MUST NOT fall back to +legacy admission. Existing snapshots remain readable; legacy Python proxy blends +require identifiable fresh contributors before a new analysis. Explicitly malformed clocks MUST be excluded from count-based region, sunspot and flare proxies as well as numeric signals. Python row-time parsing retains only native-supported calendar date/whole-second legacy forms and full explicit-UTC `Z`/`+00:00` forms with diff --git a/docs/TRANSACTIONAL_FEED.md b/docs/TRANSACTIONAL_FEED.md index 2f97d870..009f534f 100644 --- a/docs/TRANSACTIONAL_FEED.md +++ b/docs/TRANSACTIONAL_FEED.md @@ -17,6 +17,15 @@ Reusing a fixture after a failed refresh keeps its `fixture` origin, original cl and bytes; the new acquisition records the failure/degradation separately. Reuse cannot promote fixture data to observed cached data. +HTTP success is not observation availability. Acquisition rejects provider error +envelopes and non-object rows in known tabular products. Each critical RTSW product +must contain at least one finite numeric measurement with an admitted source clock +and `active` not explicitly false. Empty region, sunspot and flare-event catalogues +are legitimate optional absences. Rejected current bytes cannot become healthy +source products: attributable fallback is labelled degraded, and `--fail-on-degraded` +retains the last derived pointer. These checks do not establish scientific accuracy +or replace the separate observation-age assessment. + Native source-pointer intake and Python daily derivation carry the validated product source into row selection and normalized-frame provenance. When an F10.7 row omits `source`, the manifest attribution is used without changing @@ -29,6 +38,15 @@ select the same newest eligible numeric row; signal freshness follows that selec row rather than a newer nonnumeric record. Invalid-only inputs retain raw report/source metadata but cannot supply a numeric signal or activity proxy. +New Python reports distinguish their illustrative `activity_index` default from +`activity_observation`: the latter has an availability status, nullable value and +explicit contributing source IDs/values. Only fresh attributable activity contributors +can authorize scalar assimilation; fresh wind/magnetic context cannot authorize a +missing or stale activity proxy. An unavailable descriptor retains the simulation's +prior activity and variance. Older reports remain readable, while legacy Python +activity blends with ambiguous contributor freshness are withheld from new analysis. +See [activity admission corrections](review_activity_fix.md) for compatibility and tests. + `research-data-bundle.v1` binds its source manifest digest and all snapshot, normalized-observation, feed-status, series-manifest, and available-series-frame components. Explicit unavailable series slots retain their original index/time and diff --git a/docs/review_activity_fix.md b/docs/review_activity_fix.md new file mode 100644 index 00000000..7fbc86ff --- /dev/null +++ b/docs/review_activity_fix.md @@ -0,0 +1,58 @@ +# Review correction: activity evidence and daily stage + +The Python daily producer now emits `observed_context.activity_observation` separately +from the illustrative `activity_index` used to choose fixture density. The latter +retains its existing 0.9 fallback when no proxy is present; that fallback is not an +observation and cannot reduce analysis uncertainty. + +The additive descriptor contains `status` (`available` or `unavailable`), a nullable +`value`, and an ordered list of `{id, value}` contributors. Eligible sources are +solar-region counts, sunspot-report counts, recent flare counts, and F10.7 (daily +before the existing monthly fallback). Inputs must have attributable provenance, +known nonfuture timestamps within their freshness limit, and not be explicitly +inactive. Count contributions use only qualifying fresh rows, not older history in +the same payload. RTSW wind/magnetic context, Kp, and X-ray flux are not activity +contributors. Contributions and their mean are rounded to six decimal places; +the scalar remains an illustrative proxy, not an empirically calibrated quantity. + +Rust CLI assimilation requires a nonempty available descriptor, supported unique +contributor identities, matching admitted observation frames, known fresh ages, +and a value consistent with the mean of contributor values. Unavailable or malformed +descriptors preserve the synthetic prior and variance and explain why. The accepted +report and its observed context are retained unchanged for audit. + +Existing historical snapshots remain readable. Older Python reports with count +metadata but no descriptor can be assimilated only when all of their actual proxy +contributors have attributable frames and known fresh ages. A missing proxy does +not inherit wind/magnetic freshness. Ambiguous historical daily/monthly F10.7 +attribution is withheld instead of guessed. Existing native single-signal and +historical simple observation reports retain their prior explicit scalar contract. +No committed historical bundle was regenerated or relabelled. + +Daily fixture `learning.cycle_stage` now follows the same instantaneous thresholds +as Rust and the browser: below 0.45 is solar minimum, 0.45 through below 0.75 is +rising or declining phase, and 0.75 onward is solar maximum. This does not alter +phase-based synthetic cycle-series labels or claim that a scalar determines an +observed solar-cycle phase. Literal threshold cases are shared in +`tests/fixtures/activity-stage-cases.json`. + +Validation commands: + +```text +python -m unittest discover -s tests/python -p test_activity_observation.py -v +python -m unittest discover -s tests/python -p test_bundle_observation_provenance.py -v +python -m unittest discover -s tests/python -p test_future_freshness.py -v +cargo test -p solar-cli --locked +``` + +CI builds the native CLI and sets `SOL_REQUIRE_CLI=1` before the Python activity +suite, so its producer-to-CLI checks cannot silently skip. The round-trip uses +explicit fresh, missing and stale synthetic activity reports; it checks mode, +activity, variance, shared snapshot admission and research-only readiness after +48 transport steps. It does not require the app's historical report to qualify +for new assimilation. + +The missing-proxy end-to-end reproduction now keeps a 0.2 prior at 0.2 with variance +0.04 and mode Synthetic, where the earlier implementation emitted 0.76, variance +approximately 0.008, and mode Assimilation. These are offline synthetic regression +results, not upstream or independent scientific qualification. diff --git a/services/ephemeris-server/README.md b/services/ephemeris-server/README.md index aa70d1a5..4289a90c 100644 --- a/services/ephemeris-server/README.md +++ b/services/ephemeris-server/README.md @@ -25,6 +25,11 @@ python services/ephemeris-server/server.py --host 127.0.0.1 --port 8787 `ssd.jpl.nasa.gov`** (JPL Horizons); there are no Python dependencies and no kernel files to download. +Run from a complete repository checkout: cache admission imports the existing +`tools/validate_ephemeris_snapshot.py` validator and its `docs/` schema by paths +relative to this script, independent of the current working directory. Copying +`server.py` alone is not a supported standalone installation. + - `GET /health` → `{ "status": "ok", "provider": "horizons-de441", ... }` - `GET /v3/sky?unix=&lat=&lon=&elev=` → `ephemeris-snapshot.v3` - `GET /v1/sky?...` and `/v2/sky?...` → HTTP 409 with an explicit v3 upgrade explanation @@ -35,6 +40,19 @@ for an instant is subject to the 20-second overall deadline; no current live lat the URL the frontend uses with `window.SOL_EPHEMERIS_SERVER`. There is no default remote endpoint; the user explicitly consents to sending the selected location/time to the displayed recipient. +Cache reads are bounded to 1 MiB and admitted through the full strict v3 contract, +including the requested epoch and observer. The internal `ephemeris-cache.v1` +envelope also binds all four exact hexadecimal float request values, because +different Unix floats can round to the same Julian day. The public HTTP response +is still the unwrapped `ephemeris-snapshot.v3` object. + +Malformed, oversized or mismatched entries are discarded and rebuilt once through +the bounded provider path; a failed or invalid rebuild is an error, not a cache hit. +The cache namespace is `v7`; older `v6` entries are ignored and rebuilt lazily. +There is no bulk cache deletion. Rollback to an earlier server revision restores +that revision's cache namespace and its older admission behavior; it does not +change observed-source data or browser-selected snapshots. + ## Contract notes - Bodies and field names match the WASM engine's snapshot; `provider`/`accuracy` blocks mark diff --git a/services/ephemeris-server/server.py b/services/ephemeris-server/server.py index 4ec6f23b..1cb7f592 100644 --- a/services/ephemeris-server/server.py +++ b/services/ephemeris-server/server.py @@ -35,6 +35,7 @@ import re import select import socket +import sys import tempfile import time import threading @@ -43,13 +44,22 @@ import urllib.request from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Any, Callable +# The documented provider runs from the repository, sharing its authoritative +# stdlib-only validator and schema instead of maintaining another v3 contract. +TOOLS = Path(__file__).resolve().parents[2] / "tools" +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) +from validate_ephemeris_snapshot import parse_snapshot, validate as validate_snapshot + AU_KM = 149_597_870.7 EARTH_R_KM = 6378.14 HORIZONS = "https://ssd.jpl.nasa.gov/api/horizons.api" SCHEMA_VERSION = "ephemeris-snapshot.v3" -CACHE_VERSION = "v6" # Invalidate responses predating normalized azimuth/compass pairs. +CACHE_VERSION = "v7" # Exact-request envelope; older unbound snapshots are not reused. +CACHE_SCHEMA_VERSION = "ephemeris-cache.v1" CACHE_MAX_ENTRIES = 4096 CACHE_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "cache") USER_AGENT = "Protonmatter-Sol/0.3 (+https://github.com/Protonmatter/sol)" @@ -607,13 +617,21 @@ def snapshot_cached(unix: float, lat: float, lon: float, elev: float) -> dict[st if problem: raise ValueError(problem) path = cache_path(unix, lat, lon, elev) + request_identity = [float(value).hex() for value in (unix, lat, lon, elev)] if os.path.isfile(path): try: - with open(path, encoding="utf-8") as handle: - cached = json.load(handle) - if cached.get("schema_version") == SCHEMA_VERSION: - return cached - except (json.JSONDecodeError, OSError, AttributeError): + with open(path, "rb") as handle: + raw = handle.read(MAX_RESPONSE_BYTES + 1) + if len(raw) > MAX_RESPONSE_BYTES: + raise ValueError("cache entry exceeds byte limit") + cached = parse_snapshot(raw.decode("utf-8")) + if (isinstance(cached, dict) + and set(cached) == {"schema_version", "request", "snapshot"} + and cached["schema_version"] == CACHE_SCHEMA_VERSION + and cached["request"] == request_identity + and snapshot_matches_request(cached["snapshot"], unix, lat, lon, elev)): + return cached["snapshot"] + except (ValueError, OSError, RecursionError): pass try: os.remove(path) @@ -621,11 +639,17 @@ def snapshot_cached(unix: float, lat: float, lon: float, elev: float) -> dict[st pass snapshot = build_snapshot(unix, lat, lon, elev) + if not snapshot_matches_request(snapshot, unix, lat, lon, elev): + raise ProviderError("invalid_snapshot", "Ephemeris snapshot failed contract or request validation") + cached = {"schema_version": CACHE_SCHEMA_VERSION, "request": request_identity, "snapshot": snapshot} + raw = json.dumps(cached, separators=(",", ":"), allow_nan=False).encode("utf-8") + if len(raw) > MAX_RESPONSE_BYTES: + raise ProviderError("response_too_large", "Ephemeris snapshot exceeds cache byte limit") os.makedirs(CACHE_DIR, exist_ok=True) fd, temporary = tempfile.mkstemp(dir=CACHE_DIR, suffix=".tmp") try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - json.dump(snapshot, handle, separators=(",", ":"), allow_nan=False) + with os.fdopen(fd, "wb") as handle: + handle.write(raw) handle.flush() os.fsync(handle.fileno()) os.replace(temporary, path) @@ -638,6 +662,28 @@ def snapshot_cached(unix: float, lat: float, lon: float, elev: float) -> dict[st return snapshot +def snapshot_matches_request(snapshot: Any, unix: float, lat: float, lon: float, elev: float) -> bool: + """Validate all v3 fields, then bind its epoch and observer to this request. + + The envelope additionally preserves the exact Unix float because conversion + to a Julian day can round distinct instants to the same representable value. + """ + try: + errors = validate_snapshot(snapshot) + except (OverflowError, RecursionError): + # JSON can contain integers beyond binary64 and excessively nested + # containers. These are invalid payloads, not reusable cache entries. + # Leave contract-resource I/O and other unexpected failures visible. + return False + if errors: + return False + observer = snapshot["observer"] + return (snapshot["time"]["jd_utc"] == unix / 86_400.0 + 2_440_587.5 + and observer["terrestrial_lat_deg"] == lat + and observer["terrestrial_lon_deg_east"] == lon + and observer["elev_m"] == elev) + + def validate_params(unix: float, lat: float, lon: float, elev: float) -> str | None: if not all(math.isfinite(value) for value in (unix, lat, lon, elev)): return "unix, lat, lon, and elev must be finite numbers" diff --git a/services/ephemeris-server/test_server.py b/services/ephemeris-server/test_server.py index 4b984000..95a884af 100644 --- a/services/ephemeris-server/test_server.py +++ b/services/ephemeris-server/test_server.py @@ -4,10 +4,12 @@ from __future__ import annotations import importlib.util +import copy import io import json import math import os +import subprocess import sys import tempfile import unittest @@ -215,15 +217,16 @@ def one(_when, _lat, _lon, _elev, hid): def test_cache_round_trip_bad_entry_and_eviction(self): with tempfile.TemporaryDirectory() as tmp: with mock.patch.object(server, "CACHE_DIR", tmp), \ - mock.patch.object(server, "build_snapshot", return_value={"schema_version": server.SCHEMA_VERSION, "value": 1}) as build: + mock.patch.object(server, "definitive_positions", side_effect=fake_positions) as provider: first = server.snapshot_cached(123.9, 1, 2, 3) second = server.snapshot_cached(123.9, 1, 2, 3) self.assertEqual(first, second) - self.assertEqual(build.call_count, 1) + self.assertEqual(validator.validate(first), []) + self.assertEqual(provider.call_count, 1) path = server.cache_path(123.9, 1, 2, 3) Path(path).write_text("{bad", encoding="utf-8") server.snapshot_cached(123.9, 1, 2, 3) - self.assertEqual(build.call_count, 2) + self.assertEqual(provider.call_count, 2) for index in range(4): p = Path(tmp) / f"{index}.json" @@ -235,9 +238,159 @@ def test_cache_round_trip_bad_entry_and_eviction(self): def test_cache_preserves_exact_epoch_and_observer(self): self.assertNotEqual(server.cache_path(123.1,1,2,3), server.cache_path(123.9,1,2,3)) self.assertNotEqual(server.cache_path(123,1.00001,2,3), server.cache_path(123,1.00002,2,3)) - with tempfile.TemporaryDirectory() as tmp, mock.patch.object(server,"CACHE_DIR",tmp), mock.patch.object(server,"build_snapshot",return_value={"schema_version":server.SCHEMA_VERSION}) as build: - server.snapshot_cached(123.9,1.00001,2,3) - build.assert_called_once_with(123.9,1.00001,2,3) + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(server,"CACHE_DIR",tmp), mock.patch.object(server,"definitive_positions",side_effect=fake_positions): + snapshot = server.snapshot_cached(123.9,1.00001,2,3) + self.assertEqual(snapshot["observer"]["terrestrial_lat_deg"], 1.00001) + self.assertEqual(snapshot["observer"]["terrestrial_lon_deg_east"], 2) + self.assertEqual(snapshot["observer"]["elev_m"], 3) + self.assertEqual(validator.validate(snapshot), []) + + def test_corrupt_cache_is_replaced_once_with_a_complete_valid_snapshot(self): + args = (1_783_569_600.0, 40.71, -74.01, 12.0) + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(server, "CACHE_DIR", tmp): + with mock.patch.object(server, "definitive_positions", side_effect=fake_positions): + healthy = server.snapshot_cached(*args) + path = Path(server.cache_path(*args)) + stored = json.loads(path.read_bytes()) + + def changed_snapshot(edit): + document = copy.deepcopy(stored) + # Exercise both the legacy file and the replacement envelope during migration. + snapshot = document.get("snapshot", document) + edit(snapshot) + return json.dumps(document).encode() + + corruptions = { + "parseable truncated contract": changed_snapshot(lambda snapshot: snapshot.pop("bodies")), + "wrong version": changed_snapshot(lambda snapshot: snapshot.update(schema_version="ephemeris-snapshot.v2")), + "wrong epoch": changed_snapshot(lambda snapshot: snapshot["time"].update(jd_utc=2_461_234.5)), + "wrong latitude": changed_snapshot(lambda snapshot: snapshot["observer"].update(terrestrial_lat_deg=40.72)), + "wrong longitude": changed_snapshot(lambda snapshot: snapshot["observer"].update(terrestrial_lon_deg_east=-74.02)), + "wrong elevation": changed_snapshot(lambda snapshot: snapshot["observer"].update(elev_m=13.0)), + "nonfinite": changed_snapshot(lambda snapshot: snapshot["observer"].update(elev_m=float("nan"))), + "duplicate key": json.dumps(stored).encode().replace(b'"elev_m": 12.0', b'"elev_m": 13.0, "elev_m": 12.0'), + "overflow number": json.dumps(stored).encode().replace(b'"elev_m": 12.0', b'"elev_m": 1e999'), + "integer exceeds binary64": changed_snapshot(lambda snapshot: snapshot["time"].update(jd_tt=10 ** 400)), + "wrong time container": changed_snapshot(lambda snapshot: snapshot.update(time=[])), + "wrong observer container": changed_snapshot(lambda snapshot: snapshot.update(observer=None)), + "oversized": json.dumps(stored).encode() + b" " * (server.MAX_RESPONSE_BYTES + 1), + "invalid utf8": b"\xff", + "invalid json": b'{"schema_version":', + "excessive nesting": b"[" * 2000 + b"]" * 2000, + } + for reason, raw in corruptions.items(): + with self.subTest(reason=reason): + path.write_bytes(raw) + with mock.patch.object(server, "definitive_positions", side_effect=fake_positions) as provider: + repaired = server.snapshot_cached(*args) + self.assertEqual(repaired, healthy) + self.assertEqual(provider.call_count, 1, "corrupt entry must rebuild exactly once") + with mock.patch.object(server, "definitive_positions", side_effect=AssertionError("repaired cache missed")): + self.assertEqual(server.snapshot_cached(*args), healthy) + + def test_cache_copied_between_exact_requests_is_not_reused(self): + unix = 1_783_569_600.0 + nearby = math.nextafter(unix, math.inf) + self.assertEqual(unix / 86400 + 2440587.5, nearby / 86400 + 2440587.5) + for original, requested in [ + ((unix, 40.71, -74.01, 12.0), (nearby, 40.71, -74.01, 12.0)), + ((unix, 40.71, -74.01, 12.0), (unix, 40.71, -74.01, 13.0)), + ]: + with self.subTest(requested=requested), tempfile.TemporaryDirectory() as tmp, mock.patch.object(server, "CACHE_DIR", tmp): + with mock.patch.object(server, "definitive_positions", side_effect=fake_positions): + server.snapshot_cached(*original) + source = Path(server.cache_path(*original)) + target = Path(server.cache_path(*requested)) + target.write_bytes(source.read_bytes()) + with mock.patch.object(server, "definitive_positions", side_effect=fake_positions) as provider: + actual = server.snapshot_cached(*requested) + self.assertEqual(provider.call_count, 1, "exact request identity must match, even when JD rounds identically") + self.assertEqual(actual["observer"]["elev_m"], requested[3]) + with mock.patch.object(server, "definitive_positions", side_effect=AssertionError("healthy cache missed")): + self.assertEqual(server.snapshot_cached(*requested), actual) + + def test_corrupt_cache_rebuild_failure_does_not_return_corrupt_data(self): + args = (1_783_569_600.0, 40.71, -74.01, 12.0) + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(server, "CACHE_DIR", tmp): + path = Path(server.cache_path(*args)) + path.write_text('{"schema_version":"ephemeris-snapshot.v3","corrupt":true}', encoding="utf-8") + with mock.patch.object(server, "definitive_positions", side_effect=server.ProviderError("upstream_failed", "offline")) as provider: + with self.assertRaises(server.ProviderError): + server.snapshot_cached(*args) + self.assertEqual(provider.call_count, 1) + + def test_cache_read_is_bounded_before_parsing(self): + args = (1_783_569_600.0, 40.71, -74.01, 12.0) + reads = [] + + class BoundedReader(io.BytesIO): + def read(self, size=-1): + reads.append(size) + if size < 0 or size > server.MAX_RESPONSE_BYTES + 1: + raise AssertionError("cache read must have a finite byte budget") + return super().read(size) + + original_open = open + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(server, "CACHE_DIR", tmp): + path = Path(server.cache_path(*args)) + path.write_bytes(b" ") + + def open_cache(filename, *open_args, **kwargs): + if os.fspath(filename) == str(path): + return BoundedReader(b" " * (server.MAX_RESPONSE_BYTES + 1)) + return original_open(filename, *open_args, **kwargs) + + with mock.patch("builtins.open", side_effect=open_cache), mock.patch.object(server, "definitive_positions", side_effect=fake_positions): + rebuilt = server.snapshot_cached(*args) + self.assertEqual(reads, [server.MAX_RESPONSE_BYTES + 1]) + self.assertEqual(validator.validate(rebuilt), []) + + def test_invalid_rebuild_is_not_published_or_cached(self): + args = (1_783_569_600.0, 40.71, -74.01, 12.0) + healthy = self.build() + invalid = copy.deepcopy(healthy) + invalid["observer"]["elev_m"] = 13.0 + with tempfile.TemporaryDirectory() as tmp, mock.patch.object(server, "CACHE_DIR", tmp): + with mock.patch.object(server, "build_snapshot", return_value=invalid): + with self.assertRaisesRegex(server.ProviderError, "contract or request"): + server.snapshot_cached(*args) + self.assertEqual(list(Path(tmp).iterdir()), []) + + def test_repository_provider_imports_shared_contract_from_another_directory(self): + with tempfile.TemporaryDirectory() as tmp: + result = subprocess.run( + [sys.executable, "-I", str(Path(__file__).with_name("server.py")), "--help"], + cwd=tmp, capture_output=True, text=True, timeout=15, check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--port", result.stdout) + + def test_malformed_rebuild_reports_invalid_snapshot_without_raw_validation_errors(self): + args = (1_783_569_600.0, 40.71, -74.01, 12.0) + deep_container = [] + for _ in range(2000): + deep_container = [deep_container] + corruptions = { + "integer exceeds binary64": lambda snapshot: snapshot["time"].update(jd_tt=10 ** 400), + "wrong time container": lambda snapshot: snapshot.update(time=[]), + "wrong observer container": lambda snapshot: snapshot.update(observer=None), + "excessive nesting": lambda snapshot: snapshot.update(warnings=deep_container), + } + healthy = self.build() + for reason, corrupt in corruptions.items(): + with self.subTest(reason=reason), tempfile.TemporaryDirectory() as tmp, mock.patch.object(server, "CACHE_DIR", tmp): + invalid = copy.deepcopy(healthy) + corrupt(invalid) + with mock.patch.object(server, "build_snapshot", return_value=invalid): + with self.assertRaises(server.ProviderError) as raised: + server.snapshot_cached(*args) + self.assertEqual(raised.exception.code, "invalid_snapshot") + self.assertEqual(list(Path(tmp).iterdir()), []) + + def test_contract_resource_io_errors_are_not_classified_as_malformed_snapshot(self): + with mock.patch.object(server, "validate_snapshot", side_effect=OSError("schema unavailable")): + with self.assertRaisesRegex(OSError, "schema unavailable"): + server.snapshot_matches_request(self.build(), 1_783_569_600.0, 40.71, -74.01, 12.0) def test_request_retries_transient_errors_and_raises_permanent_errors(self): response = mock.MagicMock() diff --git a/tests/fixtures/activity-stage-cases.json b/tests/fixtures/activity-stage-cases.json new file mode 100644 index 00000000..a223d864 --- /dev/null +++ b/tests/fixtures/activity-stage-cases.json @@ -0,0 +1,10 @@ +[ + {"activity":0.2,"stage":"solar minimum"}, + {"activity":0.449999,"stage":"solar minimum"}, + {"activity":0.45,"stage":"rising or declining phase"}, + {"activity":0.6,"stage":"rising or declining phase"}, + {"activity":0.749999,"stage":"rising or declining phase"}, + {"activity":0.75,"stage":"solar maximum"}, + {"activity":0.9,"stage":"solar maximum"}, + {"activity":1.0,"stage":"solar maximum"} +] diff --git a/tests/fixtures/live-swpc-normalized.json b/tests/fixtures/live-swpc-normalized.json index c02418bb..4a1e2600 100644 --- a/tests/fixtures/live-swpc-normalized.json +++ b/tests/fixtures/live-swpc-normalized.json @@ -234,6 +234,11 @@ "generated_by": "tools/generate_fixture_snapshot.py", "observed_context": { "activity_index": 0.9, + "activity_observation": { + "contributors": [], + "status": "unavailable", + "value": null + }, "activity_proxy_sources": { "f107_cm_flux_rows": 0, "goes_xray_flares_7_day_rows": 0, diff --git a/tests/python/test_activity_observation.py b/tests/python/test_activity_observation.py new file mode 100644 index 00000000..72fe4d41 --- /dev/null +++ b/tests/python/test_activity_observation.py @@ -0,0 +1,157 @@ +"""Actual producer regressions for observation availability and daily stage labels.""" +from __future__ import annotations +import json +from pathlib import Path +import sys +import os +import subprocess +import tempfile +import unittest +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tools')) +import data_bundles as bundles +import generate_fixture_snapshot as generator +from validate_operational_readiness import validate_readiness +from validate_snapshot import loads_strict, validate + +NOW = '2026-09-28T12:00:00Z' + +class ActivityObservationTests(unittest.TestCase): + def cli_binary(self): + binary = ROOT / 'target/debug' / ('solar-cli.exe' if os.name == 'nt' else 'solar-cli') + if not binary.is_file(): + message = 'build solar-cli with cargo build -p solar-cli --locked first' + if os.environ.get('SOL_REQUIRE_CLI') == '1': + self.fail(message) + self.skipTest(message) + return binary + + def report(self, optional_rows=None): + with tempfile.TemporaryDirectory(prefix='sol-activity-') as directory: + payloads = { + 'rtsw_mag_1m.json': [{'time_tag': NOW, 'bz_gsm': -2}], + 'rtsw_wind_1m.json': [{'time_tag': NOW, 'speed': 400}], + **(optional_rows or {}), + } + products = [dict(product_id=name, source='synthetic regression source', + origin='current-fetch', observation_time_utc=NOW, retrieved_at_utc=NOW, + quality=['offline synthetic regression'], failure=None, license='synthetic', + critical=name.startswith('rtsw'), payload=bundles.json_bytes(rows)) + for name, rows in payloads.items()] + source = bundles.create_source_bundle(Path(directory), bundle_id='source', + acquired_at_utc=NOW, products=products) + return generator.build_bundle_observation_report(source, evaluated_at_utc=NOW) + + def test_missing_activity_proxies_are_unavailable_despite_fresh_rtsw(self): + context = self.report()['observed_context'] + self.assertEqual(context['activity_index'], 0.9, 'retain illustrative fixture default') + self.assertEqual(context.get('activity_observation'), + {'status': 'unavailable', 'value': None, 'contributors': []}) + + def test_stale_future_unknown_and_inactive_f107_cannot_supply_activity(self): + cases = [ + {'time_tag': '2026-01-01T00:00:00Z', 'flux': 150}, + {'time_tag': '2026-09-28T12:00:01Z', 'flux': 150}, + {'flux': 150}, + {'time_tag': NOW, 'flux': 150, 'active': False}, + {'time_tag': NOW, 'flux': 150, 'source': 'unknown'}, + ] + for row in cases: + with self.subTest(row=row): + context = self.report({'f107_cm_flux.json': [row]})['observed_context'] + self.assertEqual(context.get('activity_observation'), + {'status': 'unavailable', 'value': None, 'contributors': []}) + + def test_fresh_f107_uses_actual_contributor_and_excludes_stale_count_proxy(self): + context = self.report({ + 'f107_cm_flux.json': [{'time_tag': NOW, 'flux': 150}], + 'solar_regions.json': [{'time_tag': '2026-01-01T00:00:00Z', 'region': 1}], + })['observed_context'] + self.assertEqual(context.get('activity_observation'), {'status': 'available', + 'value': 0.5, 'contributors': [{'id': 'swpc-f107-cm-flux', 'value': 0.5}]}) + + def test_count_proxy_uses_only_fresh_rows(self): + context = self.report({'solar_regions.json': + [{'time_tag': NOW, 'region': 1}] + + [{'time_tag': '2026-01-01T00:00:00Z', 'region': i} for i in range(20)] + })['observed_context'] + self.assertEqual(context.get('activity_observation'), {'status': 'available', + 'value': 0.25, 'contributors': [{'id': 'swpc-solar-regions', 'value': 0.25}]}) + + def test_count_proxy_evidence_uses_an_eligible_active_row(self): + report = self.report({'solar_regions.json': [ + {'time_tag': NOW, 'region': 1, 'active': False}, + {'time_tag': NOW, 'region': 2, 'active': True}, + ]}) + self.assertEqual(report['observed_context']['activity_observation']['status'], 'available') + frame = next(frame for frame in report['frames'] if frame['id'] == 'swpc-solar-regions') + self.assertIs(frame['provenance']['active'], True) + + def test_produced_count_proxy_reaches_cli_analysis(self): + binary = self.cli_binary() + report = self.report({'solar_regions.json': [ + {'time_tag': NOW, 'region': 1, 'active': False}, + {'time_tag': NOW, 'region': 2, 'active': True}, + ]}) + with tempfile.TemporaryDirectory(prefix='sol-activity-cli-') as directory: + report_path = Path(directory)/'report.json' + output = Path(directory)/'snapshot.json' + report_path.write_text(json.dumps(report)) + result = subprocess.run([str(binary), 'simulate', '--steps', '0', '--activity', '0.9', + '--observations', str(report_path), '--out', str(output)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + snapshot = loads_strict(output.read_text(encoding='utf-8')) + self.assertEqual(snapshot['run']['mode'], 'Assimilation') + self.assertEqual(snapshot['run']['activity_index'], 0.38) + + def test_producer_cli_roundtrip_validates_analysis_and_withheld_modes(self): + binary = self.cli_binary() + # Literal expectations: K=.04/(.04+.01)=.8. Fresh F10.7=150 yields + # activity .5, so prior .2 becomes .44 with variance .008. Missing or + # stale activity must leave both prior values unchanged, even after 48h. + cases = ( + ('fresh', {'f107_cm_flux.json': [{'time_tag': NOW, 'flux': 150}]}, 'Assimilation', 0.44, 0.008), + ('missing', {}, 'Synthetic', 0.2, 0.04), + ('stale', {'f107_cm_flux.json': [{'time_tag': '2026-01-01T00:00:00Z', 'flux': 150}]}, 'Synthetic', 0.2, 0.04), + ) + with tempfile.TemporaryDirectory(prefix='sol-activity-roundtrip-') as directory: + for name, payloads, mode, activity, variance in cases: + with self.subTest(case=name): + report = self.report(payloads) + report_path = Path(directory) / f'{name}-observations.json' + output = Path(directory) / f'{name}-snapshot.json' + report_path.write_text(json.dumps(report), encoding='utf-8') + result = subprocess.run([str(binary), 'simulate', '--steps', '48', + '--dt-hours', '1', '--seed', '42', '--activity', '0.2', + '--observations', str(report_path), '--out', str(output)], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + snapshot = loads_strict(output.read_text(encoding='utf-8')) + self.assertEqual(snapshot['run']['mode'], mode) + self.assertEqual(snapshot['run']['activity_index'], activity) + self.assertAlmostEqual(snapshot['uncertainty']['activity']['variance'], variance, places=7) + self.assertEqual(validate(snapshot), []) + self.assertEqual(validate_readiness(snapshot['operational_readiness'], False), []) + + def test_monthly_fallback_evidence_uses_the_active_numeric_contributor(self): + report = self.report({'observed-solar-cycle-indices.json': [ + {'time_tag': NOW, 'f10.7': 235, 'active': False}, + {'time_tag': '2026-09-28T11:00:00Z', 'f10.7': 150, 'active': True}, + ]}) + self.assertEqual(report['observed_context']['activity_observation'], {'status': 'available', + 'value': 0.5, 'contributors': [{'id': 'swpc-observed-cycle-indices', 'value': 0.5}]}) + frame = next(frame for frame in report['frames'] if frame['id'] == 'swpc-observed-cycle-indices') + self.assertIs(frame['provenance']['active'], True) + self.assertEqual(frame['provenance']['time_tag'], '2026-09-28T11:00:00Z') + + def test_daily_stage_matches_shared_literal_threshold_cases(self): + report = self.report() + cases = json.loads((ROOT/'tests/fixtures/activity-stage-cases.json').read_text()) + for case in cases: + with self.subTest(activity=case['activity']): + report['observed_context']['activity_index'] = case['activity'] + snapshot = generator.build_snapshot(42, 8, 4, report) + self.assertEqual(snapshot['learning']['cycle_stage'], case['stage']) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/python/test_feed_commands.py b/tests/python/test_feed_commands.py index 62b54efc..043007dc 100644 --- a/tests/python/test_feed_commands.py +++ b/tests/python/test_feed_commands.py @@ -72,8 +72,8 @@ def test_fetch_entrypoint_and_transport_success_failure_size(self): with self.assertRaises(Exception):fetch.fetch("https://example.invalid/",timeout_seconds=1,attempts=1) for raw in (b"{}",b"[]",b"null",b"not JSON",b"[NaN]"): with self.subTest(raw=raw),self.assertRaises(ValueError):fetch.validate_payload("x.json",raw) - self.assertEqual(fetch.display_path(ROOT/"tools/x.json"),"tools/x.json") - self.assertEqual(fetch.display_path(self.root/"x.json"),"x.json") + self.assertEqual(generate.display_path(ROOT/"tools/x.json"),"tools/x.json") + self.assertEqual(generate.display_path(self.root/"x.json"),"x.json") def test_critical_fetch_failure_keeps_old_pointer_and_records_failure(self): source=self.source();before=(self.root/"source/current.json").read_bytes() diff --git a/tests/python/test_feed_transaction.py b/tests/python/test_feed_transaction.py index d55a7554..86f1105a 100644 --- a/tests/python/test_feed_transaction.py +++ b/tests/python/test_feed_transaction.py @@ -22,7 +22,7 @@ def setUp(self): def test_fetch_fallback_preserves_original_time_and_payload(self): endpoint=acquisition.Endpoint("rtsw_mag_1m.json","https://example.invalid/fixture","NOAA fixture","daily","observed",True,"fixture") - with mock.patch.object(acquisition,"build_endpoints",return_value=[endpoint]), mock.patch.object(acquisition,"fetch",return_value=b'[{"time_tag":"2026-09-10T01:00:00Z","source":"NOAA","active":true}]'): + with mock.patch.object(acquisition,"build_endpoints",return_value=[endpoint]), mock.patch.object(acquisition,"fetch",return_value=b'[{"time_tag":"2026-09-10T01:00:00Z","source":"NOAA","active":true,"bz_gsm":-2}]'): first=acquisition.acquire_bundle(self.root,bundle_id="first",stamp=date(2026,9,11),acquired_at_utc="2026-09-11T00:00:00Z") with mock.patch.object(acquisition,"build_endpoints",return_value=[endpoint]), mock.patch.object(acquisition,"fetch",side_effect=OSError("offline")): second=acquisition.acquire_bundle(self.root,bundle_id="second",stamp=date(2026,9,12),acquired_at_utc="2026-09-12T00:00:00Z") diff --git a/tests/python/test_series_serialization.py b/tests/python/test_series_serialization.py new file mode 100644 index 00000000..98295f70 --- /dev/null +++ b/tests/python/test_series_serialization.py @@ -0,0 +1,40 @@ +"""Raw-byte contract for portable synthetic-series serialization.""" +from __future__ import annotations + +import contextlib +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) +import generate_series + + +class SeriesSerializationTests(unittest.TestCase): + def test_generated_manifest_uses_canonical_lf_bytes(self): + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) + arguments = ["generate_series.py", "--base", str(ROOT / "apps/web/data/latest-state.json"), + "--out-dir", str(output), "--frames", "1", "--lon-count", "4", "--lat-count", "3"] + with mock.patch.object(sys, "argv", arguments), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(generate_series.main(), 0) + raw = (output / "manifest.json").read_bytes() + self.assertNotIn(b"\r", raw, "generated manifest must not depend on platform newline translation") + self.assertTrue(raw.endswith(b"\n")) + expected = { + "schema_version": "series-manifest.v1", "months_span": 132.0, + "note": "Deterministic synthetic solar-cycle series; latitudes follow an idealized butterfly diagram.", + "frames": [{"index": 0, "file": "frame-00.json", "phase": 0.0, "stage": "solar minimum", + "activity_index": 0.2, "months": 0.0, "region_count": 10}], + } + self.assertEqual(raw, (json.dumps(expected, indent=2, sort_keys=True) + "\n").encode("utf-8")) + self.assertFalse(list(output.glob("*.tmp"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_smoke_server_connections.py b/tests/python/test_smoke_server_connections.py new file mode 100644 index 00000000..112b2e97 --- /dev/null +++ b/tests/python/test_smoke_server_connections.py @@ -0,0 +1,42 @@ +"""A Chromium speculative connection must not block the local smoke server.""" +from __future__ import annotations + +import socket +import sys +import tempfile +import threading +import unittest +import urllib.request +from pathlib import Path +from unittest import mock +from urllib.parse import urlsplit + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tools")) +import browser_smoke + + +class SmokeServerConnectionTests(unittest.TestCase): + def test_idle_preconnected_socket_does_not_block_page_request(self): + accepted = threading.Event() + original = browser_smoke.QuietHandler.handle + + def observe_accept(handler): + accepted.set() + original(handler) + + with tempfile.TemporaryDirectory(prefix="sol-smoke-connections-") as directory: + root = Path(directory) + (root / "index.html").write_text("smoke fixture", encoding="utf-8") + with mock.patch.object(browser_smoke.QuietHandler, "handle", observe_accept): + with browser_smoke.serve(root) as base: + accepted.clear() + address = urlsplit(base) + with socket.create_connection((address.hostname, address.port), timeout=2): + self.assertTrue(accepted.wait(2), "idle connection was not accepted") + with urllib.request.urlopen(base + "/index.html", timeout=2) as response: + self.assertEqual(response.status, 200) + self.assertEqual(response.read(), b"smoke fixture") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_source_payloads.py b/tests/python/test_source_payloads.py new file mode 100644 index 00000000..01b4013a --- /dev/null +++ b/tests/python/test_source_payloads.py @@ -0,0 +1,116 @@ +"""Provider transport success must not be confused with usable observations.""" +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import unittest +from datetime import date +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools")) + +import data_bundles as bundles +import fetch_public_data as acquisition +import run_daily_ingest as daily + + +class SourcePayloadTests(unittest.TestCase): + def endpoints(self): + return acquisition.build_endpoints(include_jpl=False, start_date=date(2026, 9, 28))[:2] + + def payload(self, product: str, **overrides) -> bytes: + row = {"time_tag": "2026-09-28T12:00:00Z", "source": "fixture", "active": True} + row["bz_gsm" if product == "rtsw_mag_1m.json" else "proton_speed"] = -2 if product == "rtsw_mag_1m.json" else 400 + row.update(overrides) + return bundles.json_bytes([row]) + + def test_critical_products_reject_error_envelopes_and_unusable_rows(self): + for endpoint in self.endpoints(): + key = "bz_gsm" if endpoint.file == "rtsw_mag_1m.json" else "proton_speed" + invalid = [ + b'{"error":"upstream unavailable"}', b'[1,2]', b'[{}]', b'[]', + self.payload(endpoint.file, time_tag="not-a-time"), + self.payload(endpoint.file, time_tag=None), + self.payload(endpoint.file, active=False), + self.payload(endpoint.file, source="unknown"), + self.payload(endpoint.file, source=" "), + self.payload(endpoint.file, source=None), + self.payload(endpoint.file, source=True), + self.payload(endpoint.file, **{key: None}), + self.payload(endpoint.file, **{key: True}), + self.payload(endpoint.file, **{key: "Infinity"}), + self.payload(endpoint.file, **{key: "unavailable"}), + ] + for raw in invalid: + with self.subTest(product=endpoint.file, raw=raw), self.assertRaises(ValueError): + acquisition.validate_payload(endpoint.file, raw) + + def test_critical_products_accept_real_source_shapes_and_legacy_utc_clocks(self): + for endpoint in self.endpoints(): + for clock in ("2026-09-28T12:00:00Z", "2026-09-28T12:00:00", "2026-09-28T12:00:00+00:00"): + with self.subTest(product=endpoint.file, clock=clock): + acquisition.validate_payload(endpoint.file, self.payload(endpoint.file, time_tag=clock)) + fixture = ROOT / "tests/swpc_scn26_21" / endpoint.file.replace(".json", "_new.json") + acquisition.validate_payload(endpoint.file, fixture.read_bytes()) + inherited = json.loads(self.payload(endpoint.file)) + del inherited[0]["source"] # A missing field inherits endpoint attribution. + acquisition.validate_payload(endpoint.file, bundles.json_bytes(inherited)) + # A bad or inactive instrument row cannot hide an eligible row in the same product. + good = json.loads(self.payload("rtsw_mag_1m.json"))[0] + acquisition.validate_payload("rtsw_mag_1m.json", bundles.json_bytes([ + {"time_tag": "bad", "bz_gsm": None}, {**good, "active": False}, good, + ])) + + def test_optional_event_absence_is_valid_but_error_objects_are_not(self): + for product in ("solar_regions.json", "sunspot_report.json", "goes_xray_flares_7_day.json"): + with self.subTest(product=product): + acquisition.validate_payload(product, b"[]") + for raw in (b'{"error":"maintenance"}', b'[{"error":"maintenance"}]', b'[null]'): + with self.assertRaises(ValueError): + acquisition.validate_payload(product, raw) + for product in ("helioviewer_datasources.json", "jpl_horizons_sun_earth.json"): + with self.subTest(product=product), self.assertRaises(ValueError): + acquisition.validate_payload(product, b'{"error":"maintenance"}') + + def test_invalid_current_payload_keeps_source_and_derived_selection(self): + with tempfile.TemporaryDirectory(prefix="sol-payload-") as tmp: + root = Path(tmp) + endpoints = self.endpoints() + with mock.patch.object(acquisition, "build_endpoints", return_value=endpoints), mock.patch.object( + acquisition, "fetch", side_effect=[self.payload(e.file) for e in endpoints] + ): + source = acquisition.acquire_bundle(root / "source", bundle_id="accepted", stamp=date(2026, 9, 28), acquired_at_utc="2026-09-28T12:00:00Z") + daily.derive_bundle(source, root / "derived", bundle_id="accepted", generated_at_utc="2026-09-28T12:00:00Z", seed=42) + before = (root / "derived/current.json").read_bytes() + argv = ["daily", "--cache", str(root / "source"), "--web-data", str(root / "derived"), "--fail-on-degraded"] + with mock.patch.object(sys, "argv", argv), mock.patch.object(acquisition, "build_endpoints", return_value=endpoints), mock.patch.object( + acquisition, "fetch", return_value=b'{"error":"upstream unavailable"}' + ), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(daily.main(), 1) + self.assertEqual((root / "derived/current.json").read_bytes(), before) + retained = bundles.resolve_source_bundle(root / "source/current.json") + self.assertEqual([c.raw for c in retained.components], [c.raw for c in source.components]) + products = json.loads(retained.manifest_raw)["products"] + self.assertTrue(all(p["origin"] == "cached-fallback" and p["failure"] for p in products)) + self.assertTrue(list((root / "derived/attempts").glob("*.json"))) + + def test_invalid_first_acquisition_cannot_select_a_candidate(self): + with tempfile.TemporaryDirectory(prefix="sol-payload-") as tmp: + root = Path(tmp) + argv = ["daily", "--cache", str(root / "source"), "--web-data", str(root / "derived"), "--fail-on-degraded"] + with mock.patch.object(sys, "argv", argv), mock.patch.object(acquisition, "build_endpoints", return_value=self.endpoints()), mock.patch.object( + acquisition, "fetch", return_value=b'{"error":"upstream unavailable"}' + ), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(daily.main(), 1) + self.assertFalse((root / "source/current.json").exists()) + self.assertFalse((root / "derived/current.json").exists()) + self.assertTrue(list((root / "derived/attempts").glob("*.json"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/web/activityStageParity.test.mjs b/tests/web/activityStageParity.test.mjs new file mode 100644 index 00000000..1eb3de15 --- /dev/null +++ b/tests/web/activityStageParity.test.mjs @@ -0,0 +1,9 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import test from 'node:test'; +import {stageFromActivity} from '../../apps/web/js/format.js'; + +const cases=JSON.parse(fs.readFileSync(new URL('../fixtures/activity-stage-cases.json',import.meta.url),'utf8')); +for(const {activity,stage} of cases)test(`browser activity ${activity} agrees with the shared cycle-stage contract`,()=>{ + assert.equal(stageFromActivity(activity),stage); +}); diff --git a/tests/web/orreryContextRecovery.test.mjs b/tests/web/orreryContextRecovery.test.mjs index 95640160..e3273dda 100644 --- a/tests/web/orreryContextRecovery.test.mjs +++ b/tests/web/orreryContextRecovery.test.mjs @@ -10,7 +10,7 @@ test('restoring a paused graphics context reuses retained physical metadata with const requests=h.requests.length,positionComputations=h.positionEpochs.length,contexts=h.contexts,draws=h.draws; // Model a tab which has remained paused far longer than the ordinary metadata // refresh cadence. Restoring its graphics is not a new scientific time intent. - h.advanceMonotonicTime(20000);h.holdSnapshots(); + h.advanceMonotonicTime(20000);h.setWallUnix(1800007200);h.holdSnapshots(); h.event('orreryCanvas','webglcontextlost'); h.event('orreryCanvas','webglcontextrestored');await h.settleCatalogues(); assert.equal(h.requests.length,requests,'graphics restoration must not queue a deadline-bound metadata request'); diff --git a/tests/web/orreryTimePersistence.test.mjs b/tests/web/orreryTimePersistence.test.mjs new file mode 100644 index 00000000..44f97368 --- /dev/null +++ b/tests/web/orreryTimePersistence.test.mjs @@ -0,0 +1,116 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import {orreryHarness} from './helpers/orreryHarness.mjs'; + +// These exercise production entry/clock/detail logic. Only browser/GPU/engine I/O +// and the two clocks are provided by the existing harness. +test('paused System reentry retains its scientific epoch after wall time passes', async t => { + const h=await orreryHarness(t,{controls:true}); + await h.enterOrrery();h.setAnimate(false); + const epoch=h.state.renderUnix,bodies=JSON.stringify(h.state.bodies); + h.leaveOrrery();h.setWallUnix(1800003600); + await h.enterOrrery();h.frame(1000); + assert.equal(h.state.renderUnix,epoch); + assert.equal(JSON.stringify(h.state.bodies),bodies); + assert.equal(h.state.animate,false);assert.equal(h.frames.size,0); + // Explicit resume advances from that preserved epoch, not the newer wall clock. + h.setAnimate(true);h.frame(2000); + assert.equal(h.state.renderUnix,epoch+57.6); + h.leaveOrrery(); +}); + +test('running System reentry resumes from its last simulated epoch without wall-time drift', async t => { + const h=await orreryHarness(t,{controls:true}); + await h.enterOrrery();h.frame(1000); + const epoch=h.state.renderUnix; + h.leaveOrrery();h.setWallUnix(1800007200); + await h.enterOrrery(); + assert.equal(h.state.renderUnix,epoch); + h.frame(2000);assert.equal(h.state.renderUnix,epoch+57.6); + h.leaveOrrery(); +}); + +test('paused failed reentry and full Retry retain the accepted epoch', async t => { + const h=await orreryHarness(t,{controls:true}); + await h.enterOrrery();h.setAnimate(false); + const epoch=h.state.renderUnix; + h.leaveOrrery();h.setWallUnix(1800003600);h.failWorker(true); + await h.enterOrrery();await h.settle(); + assert.equal(h.state.renderUnix,epoch); + assert.equal(h.nodes.orreryCanvas.style.display,'none'); + h.setWallUnix(1800007200);h.failWorker(false);await h.retry(); + assert.equal(h.state.renderUnix,epoch); + assert.equal(h.nodes.orreryCanvas.style.display,''); + assert.equal(h.state.animate,false); + h.frame(1000);assert.equal(h.state.renderUnix,epoch);h.leaveOrrery(); +}); + +test('Now and explicit time offsets still replace a retained paused System epoch', async t => { + const h=await orreryHarness(t,{controls:true}); + await h.enterOrrery();h.setAnimate(false);h.leaveOrrery(); + h.setWallUnix(1800003600);await h.enterOrrery(); + assert.equal(h.state.renderUnix,1800000000); + h.now();await h.settle();assert.equal(h.state.renderUnix,1800003600); + h.input('orreryTime','1');await h.settle(); + assert.equal(h.state.renderUnix,1831561200); + h.leaveOrrery();h.setWallUnix(1800007200);await h.enterOrrery(); + assert.equal(h.state.renderUnix,1831561200); + assert.equal(h.state.animate,false);h.leaveOrrery(); +}); + +function selectSmallBody(h,name) { + h.input('orrerySearch',name); + const row=h.nodes.orreryPositions.children.find(node=>node.dataset.objectId===name); + assert.ok(row,`${name} remains keyboard-selectable`);row.click(); +} +function distanceCell(h) { + const cells=h.nodes.orreryDetail.querySelector('dl').children; + return cells[cells.findIndex(node=>node.textContent==='Distance from Sun')+1]; +} + +// Recorded distances from the existing illustrative catalogue at fixed epochs. +// These check synchronization of the displayed facts, not orbital accuracy. +for(const [name,before,after] of [ + ['1P/Halley','34.92 AU · light 4.8 h','2.45 AU · light 0.3 h'], + ['Pluto','35.62 AU · light 4.9 h','43.78 AU · light 6.1 h'], +])test(`changing System time refreshes ${name} distance without replacing focused details`,async t=>{ + const h=await orreryHarness(t,{controls:true});await h.enterOrrery();h.setAnimate(false); + selectSmallBody(h,name);const cell=distanceCell(h); + assert.equal(cell.textContent,before); + const disclosure=h.nodes.orreryDetail.querySelector('details'),summary=disclosure.querySelector('summary'); + disclosure.open=true;summary.focus(); + h.input('orreryTime','34');await h.settle(); + assert.equal(cell.textContent,after); + assert.equal(distanceCell(h),cell); + assert.equal(h.nodes.orreryDetail.querySelector('details'),disclosure); + assert.equal(disclosure.open,true);assert.equal(summary.ownerDocument.activeElement,summary); + h.leaveOrrery(); +}); + +test('animation refreshes selected comet facts before another metadata response',async t=>{ + const h=await orreryHarness(t,{controls:true});await h.enterOrrery();h.setAnimate(false); + h.input('orreryTime','34');await h.settle();selectSmallBody(h,'1P/Halley'); + const cell=distanceCell(h);assert.equal(cell.textContent,'2.45 AU · light 0.3 h'); + h.input('orrerySpeedEntry','18262.5'); // 50 years/s, existing bounded maximum. + h.holdSnapshots();h.setAnimate(true);h.frame(1000); + assert.equal(cell.textContent,'2.62 AU · light 0.4 h'); + assert.equal(distanceCell(h),cell);h.leaveOrrery(); +}); + +test('selected small-body facts follow explicit time changes while graphics are unavailable',async t=>{ + const h=await orreryHarness(t,{controls:true});await h.enterOrrery();h.setAnimate(false); + selectSmallBody(h,'1P/Halley');const cell=distanceCell(h); + h.event('orreryCanvas','webglcontextlost'); + h.input('orreryTime','34');await h.settle(); + assert.equal(cell.textContent,'2.45 AU · light 0.3 h'); + assert.equal(distanceCell(h),cell);h.leaveOrrery(); +}); + +test('probe cards keep their disclosed fixed approximate distance when System time changes',async t=>{ + const h=await orreryHarness(t,{controls:true});await h.enterOrrery();h.setAnimate(false); + selectSmallBody(h,'Voyager 1');const cell=distanceCell(h); + assert.equal(cell.textContent,'167.00 AU · light 23.1 h'); + h.input('orreryTime','34');await h.settle(); + assert.equal(cell.textContent,'167.00 AU · light 23.1 h'); + assert.equal(distanceCell(h),cell);h.leaveOrrery(); +}); diff --git a/tools/browser_smoke.py b/tools/browser_smoke.py index ee641a93..ddd843c6 100644 --- a/tools/browser_smoke.py +++ b/tools/browser_smoke.py @@ -15,7 +15,6 @@ import os import re import shutil -import socketserver import struct import subprocess import tempfile @@ -38,6 +37,10 @@ class QuietHandler(http.server.SimpleHTTPRequestHandler): + # Chromium can open a speculative socket before sending a request. Bound its + # idle lifetime; other requests are served on separate daemon threads. + timeout = 10 + def __init__(self, *args, base_path="/", namespace="", **kwargs): self.base_path = base_path self.namespace = namespace @@ -294,7 +297,7 @@ def serve(directory: Path, base_path: str = "/", namespace: str = ""): handler = lambda *args, **kwargs: QuietHandler( # noqa: E731 *args, directory=str(directory), base_path=base_path, namespace=namespace, **kwargs ) - with socketserver.TCPServer(("127.0.0.1", 0), handler) as server: + with http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) as server: thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() port = server.server_address[1] @@ -302,9 +305,10 @@ def serve(directory: Path, base_path: str = "/", namespace: str = ""): deadline = time.monotonic() + 5.0 while True: try: - urllib.request.urlopen( + with urllib.request.urlopen( f"http://127.0.0.1:{port}{base_path}{namespace}index.html", timeout=1 - ).read(1) + ) as response: + response.read() break except OSError: if time.monotonic() >= deadline: diff --git a/tools/ephemeris-data/stars/README.md b/tools/ephemeris-data/stars/README.md index 7036e708..15e7d668 100644 --- a/tools/ephemeris-data/stars/README.md +++ b/tools/ephemeris-data/stars/README.md @@ -6,7 +6,7 @@ is reproducible offline forever; do not edit these by hand. | File | Upstream | License | sha256 | |---|---|---|---| -| `hipparcos_mag7.json` | [`star-catalog` crate v0.2.0](https://crates.io/crates/star-catalog) (`hipparcos_mag7.json`, unmodified) | MIT OR Apache-2.0 | `392f86c4cb52f13976b667e08d86686e390481a50349e6130e245bd287131368` | +| `hipparcos_mag7.json` | [`star-catalog` crate v0.2.0 source archive](https://docs.rs/crate/star-catalog/0.2.0/source/) (`hipparcos_mag7.json`, unmodified) | MIT OR Apache-2.0 | `392f86c4cb52f13976b667e08d86686e390481a50349e6130e245bd287131368` | | `hipparcos_names.json` | same crate (`names.json`, unmodified) | MIT OR Apache-2.0 | `e227dcd897c7a67b06e6b4640f850b050ca0a0119bcdd08e8694d721d2a50017` | | `d3celestial_starnames.json` | [`d3-celestial` npm v0.7.35](https://www.npmjs.com/package/d3-celestial) (`data/starnames.json`, unmodified) | BSD-3-Clause | `044a73db97f45c51e194db64e2555dc295be57897bf9da90bc3781dcd17ba216` | | `pyephem_bright_stars.edb` | [`ephem` PyPI v4.2.1](https://pypi.org/project/ephem/) — the 116-line bright-star `db` block from `ephem/stars.py`, extracted verbatim | package LGPL-3.0+; entries are XEphem-format catalogue facts compiled from the Yale Bright Star Catalogue | `900c4c434144f0a03802ec6fa299e613736e18dd1dd2c8646ca3c050692e8c9d` | diff --git a/tools/fetch_public_data.py b/tools/fetch_public_data.py index 7e518f96..294a50da 100644 --- a/tools/fetch_public_data.py +++ b/tools/fetch_public_data.py @@ -12,13 +12,12 @@ import argparse import json import logging -import shutil import tempfile import time import urllib.parse import urllib.request import uuid -from dataclasses import asdict, dataclass +from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from pathlib import Path from typing import Any @@ -74,15 +73,6 @@ def acquire_bundle(cache: Path, *, bundle_id: str, stamp: date, acquired_at_utc: raise -def display_path(path: Path) -> str: - """Repo-relative POSIX path (or basename outside the repo) for manifest records that - end up committed/deployed via feed-status.json — never absolute local paths.""" - try: - return path.resolve().relative_to(REPO_ROOT).as_posix() - except (ValueError, OSError): - return path.name - - @dataclass(frozen=True) class Endpoint: file: str @@ -267,14 +257,53 @@ def fetch(url: str, *, timeout_seconds: int, attempts: int = 3) -> bytes: def validate_payload(file_name: str, raw: bytes) -> None: - """Refuse to poison the cache with a non-JSON body served under HTTP 200 (SWPC - maintenance pages do this) — the old code cached it, recorded ok=true, and the - failure surfaced later as a confusing parse error in the fixture generator.""" - if file_name.endswith(".json"): - from data_bundles import loads_strict, LIMIT - if len(raw) > LIMIT: raise ValueError("source payload exceeds size limit") - value = loads_strict(raw.decode("utf-8")) - if not isinstance(value, (dict,list)) or not value: raise ValueError("source JSON must be nonempty") + """Admit provider bytes before transport success can become source health. + + Empty optional event catalogues are legitimate. Critical RTSW feeds instead + need an eligible stamped numeric observation, using the same clock/numeric + grammar as their consumer. This checks availability, not scientific accuracy + or present-day freshness; original source bytes and clocks remain unchanged. + """ + if not file_name.endswith(".json"): + return + from data_bundles import attributable_source, loads_strict, LIMIT + from generate_fixture_snapshot import CONTEXT_NUMERIC_KEYS, numeric, parse_time_tag, row_time + + if len(raw) > LIMIT: + raise ValueError("source payload exceeds size limit") + value = loads_strict(raw.decode("utf-8")) + if not isinstance(value, (dict, list)): + raise ValueError("source JSON must be an object or row array") + records = value if isinstance(value, list) else [value] + if any(isinstance(row, dict) and any(row.get(key) for key in ("error", "errors")) for row in records): + raise ValueError("provider returned an error envelope") + + empty_event_products = {"solar_regions.json", "sunspot_report.json", "goes_xray_flares_7_day.json"} + object_products = {"helioviewer_datasources.json", "jpl_horizons_sun_earth.json"} + row_products = {endpoint.file for endpoint in build_endpoints(include_jpl=False, start_date=date(2000, 1, 1))} - object_products + if file_name in row_products: + if not isinstance(value, list) or any(not isinstance(row, dict) for row in value): + raise ValueError("source product requires an array of observation objects") + if not value and file_name in empty_event_products: + return + elif file_name in object_products and not isinstance(value, dict): + raise ValueError("source metadata product requires an object") + if not value or not any(records): + raise ValueError("source JSON must contain observations or metadata") + + if file_name in {"rtsw_mag_1m.json", "rtsw_wind_1m.json"}: + keys = CONTEXT_NUMERIC_KEYS[file_name] + if not any( + row.get("active") is not False + and ("source" not in row or attributable_source(row["source"])) + and parse_time_tag(row_time(row)) is not None + and any(numeric(row.get(key)) is not None for key in keys) + for row in value + ): + raise ValueError("critical source has no eligible timestamped numeric observation") + elif file_name == "jpl_horizons_sun_earth.json": + if not isinstance(value.get("result"), str) or not value["result"].strip(): + raise ValueError("Horizons source requires its result text") def horizons_url(start: date) -> str: @@ -296,14 +325,6 @@ def parse_date(value: str) -> date: return datetime.strptime(value, "%Y-%m-%d").date() -def atomic_write_bytes(path: Path, raw: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with tempfile.NamedTemporaryFile(delete=False, dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp") as handle: - tmp = Path(handle.name) - handle.write(raw) - tmp.replace(path) - - def write_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile("w", delete=False, dir=str(path.parent), encoding="utf-8", newline="\n", prefix=f".{path.name}.", suffix=".tmp") as handle: diff --git a/tools/generate_fixture_snapshot.py b/tools/generate_fixture_snapshot.py index 9bced0c4..0475be73 100644 --- a/tools/generate_fixture_snapshot.py +++ b/tools/generate_fixture_snapshot.py @@ -312,7 +312,7 @@ def build_snapshot(seed: int, lon_count: int, lat_count: int, observations: dict "uncertainty": illustrative_uncertainty(), "active_regions": [region_snapshot(region, 0.0) for region in active_regions], "learning": { - "cycle_stage": "solar maximum", + "cycle_stage": stage_from_activity(activity_index), "plain_language_insight": insight_from_context(observed_context), }, "observed_context": observed_context, @@ -344,6 +344,15 @@ def build_snapshot(seed: int, lon_count: int, lat_count: int, observations: dict } +def stage_from_activity(activity_index: float) -> str: + """Instantaneous illustrative stage; cycle-series phase labels remain separate.""" + if activity_index >= 0.75: + return "solar maximum" + if activity_index >= 0.45: + return "rising or declining phase" + return "solar minimum" + + def illustrative_uncertainty(time_seconds: float = 0.0) -> dict[str, Any]: """Uncalibrated scalar prior, never inferred from a spatial score or source feed.""" return {"activity": {"variance": 0.04, "units": "activity_index_squared", @@ -493,7 +502,7 @@ def candidate(name: str, descriptor: dict[str, Any] | None = None) -> dict[str, selected = latest_numeric_observation( data, *context_keys, - reject_inactive=descriptor.get("id") == "swpc-f107-cm-flux", + reject_inactive=descriptor.get("id") in ("swpc-f107-cm-flux", "swpc-observed-cycle-indices"), ) if context_keys else None if selected is not None: row = selected[1] @@ -515,6 +524,16 @@ def candidate(name: str, descriptor: dict[str, Any] | None = None) -> dict[str, def report_from_candidates(mag: dict[str, Any], wind: dict[str, Any], optional_candidates: list[dict[str, Any]]) -> dict[str, Any]: all_candidates = [mag, wind, *optional_candidates] + supplied = next((item["evaluated_at_utc"] for item in all_candidates if "evaluated_at_utc" in item), None) + now = dt.datetime.fromisoformat(supplied.replace("Z", "+00:00")) if supplied else dt.datetime.now(dt.timezone.utc) + for candidate in optional_candidates: + if candidate["id"] in ("swpc-solar-regions", "swpc-sunspot-report", "swpc-goes-xray-flares-7-day"): + contributing_rows = fresh_activity_rows(candidate, now) + if contributing_rows: + # Evidence and freshness describe the same admitted count rows. + # Retain the immutable payload; select one actual contributing row. + candidate["row"] = max(contributing_rows, key=lambda row: parse_time_tag(row_time(row))) + candidate["freshness_rows"] = contributing_rows source_mode = "cached" if any(candidate["source_mode"] == "cached" for candidate in all_candidates) else "fixture" observed_context = build_observed_context(all_candidates) return { @@ -723,7 +742,7 @@ def build_observed_context(candidates: list[dict[str, Any]]) -> dict[str, Any]: ) latest_f107 = latest_direct_f107 if latest_f107 is None: - latest_f107 = latest_numeric(cycle_rows, *CONTEXT_NUMERIC_KEYS["swpc-observed-cycle-indices"]) + latest_f107 = latest_numeric(cycle_rows, *CONTEXT_NUMERIC_KEYS["swpc-observed-cycle-indices"], reject_inactive=True) latest_kp = latest_numeric(kp_rows, *CONTEXT_NUMERIC_KEYS["swpc-planetary-k-index-1m"]) latest_xray_flux = latest_numeric(xray_rows, *CONTEXT_NUMERIC_KEYS["swpc-goes-xrays-1-day"]) latest_wind_speed = latest_numeric(rtsw_wind, *CONTEXT_NUMERIC_KEYS["rtsw_wind_1m.json"]) @@ -745,6 +764,7 @@ def build_observed_context(candidates: list[dict[str, Any]]) -> dict[str, Any]: return { "schema_version": "observed-context.v1", "activity_index": activity_index, + "activity_observation": build_activity_observation(candidates, freshness), "signal_freshness": freshness, "stale_feeds": stale_feeds, "activity_proxy_sources": { @@ -770,6 +790,56 @@ def build_observed_context(candidates: list[dict[str, Any]]) -> dict[str, Any]: } +def fresh_activity_rows(candidate: dict[str, Any], now: dt.datetime) -> list[dict[str, Any]]: + """Actual rows eligible to supply a fresh scalar activity observation.""" + if candidate.get("source_mode") != "cached": + return [] + admitted = [] + for row in context_rows(candidate.get("data")): + observed = parse_time_tag(row_time(row)) + source = row.get("source", candidate.get("manifest_source")) + if (observed is not None and attributable_source(source) and row.get("active") is not False + and 0 <= (now - observed).total_seconds() / 3600 <= FRESHNESS_LIMITS_HOURS.get(candidate["id"], 48.0)): + admitted.append(row) + return admitted + + +def build_activity_observation(candidates: list[dict[str, Any]], freshness: dict[str, Any]) -> dict[str, Any]: + """Separate a fresh attributable activity observation from fixture defaults. + + Wind, magnetic, Kp and X-ray flux context are not contributors to this proxy. + Unknown clocks, fixture-only inputs and inactive/unattributable rows cannot + reduce uncertainty. Count proxies use only fresh rows, not the feed's history. + """ + supplied = next((item["evaluated_at_utc"] for item in candidates if "evaluated_at_utc" in item), None) + now = dt.datetime.fromisoformat(supplied.replace("Z", "+00:00")) if supplied else dt.datetime.now(dt.timezone.utc) + by_id = {candidate["id"]: candidate for candidate in candidates} + contributors: list[dict[str, Any]] = [] + + def eligible_rows(identity: str) -> list[dict[str, Any]]: + candidate = by_id.get(identity, {}) + if candidate.get("source_mode") != "cached" or freshness.get(identity, {}).get("stale") is not False: + return [] + return fresh_activity_rows(candidate, now) + + for identity, divisor, offset in ( + ("swpc-solar-regions", 14.0, 0.0), + ("swpc-sunspot-report", 18.0, 0.0), + ("swpc-goes-xray-flares-7-day", 24.0, 0.45), + ): + count = len(eligible_rows(identity)) + if count: + contributors.append({"id": identity, "value": round(clamp_float(offset + count / divisor, 0.25, 1.0), 6)}) + for identity in ("swpc-f107-cm-flux", "swpc-observed-cycle-indices"): + value = latest_numeric(eligible_rows(identity), *CONTEXT_NUMERIC_KEYS[identity], reject_inactive=True) + if value is not None: + contributors.append({"id": identity, "value": round(clamp_float((value - 65.0) / 170.0, 0.25, 1.0), 6)}) + break # Daily F10.7 takes precedence over the monthly fallback. + return {"status": "available" if contributors else "unavailable", + "value": round(sum(item["value"] for item in contributors) / len(contributors), 6) if contributors else None, + "contributors": contributors} + + def rows(value: Any) -> list[dict[str, Any]]: if isinstance(value, list): return [item for item in value if isinstance(item, dict)] diff --git a/tools/generate_series.py b/tools/generate_series.py index c95d66f0..bc6a5735 100644 --- a/tools/generate_series.py +++ b/tools/generate_series.py @@ -195,7 +195,7 @@ def main() -> int: def atomic_write_text(path: Path, content: str) -> None: tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(content, encoding="utf-8") + tmp.write_text(content, encoding="utf-8", newline="\n") tmp.replace(path) diff --git a/tools/run_daily_ingest.py b/tools/run_daily_ingest.py index 390da98c..402eba6c 100644 --- a/tools/run_daily_ingest.py +++ b/tools/run_daily_ingest.py @@ -10,15 +10,10 @@ from __future__ import annotations import argparse -import json import logging -import subprocess -import sys -import tempfile import uuid from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import Any LOGGER = logging.getLogger("run_daily_ingest") REPO_ROOT = Path(__file__).resolve().parents[1]