From 7574da04b530235d34ea255b97c1da56c20edda8 Mon Sep 17 00:00:00 2001 From: Ciprian-LocalPulse Date: Wed, 23 Sep 2026 05:41:13 +0300 Subject: [PATCH 1/2] fix: tighten analysis integrity contracts --- CHANGELOG.md | 2 +- README.md | 2 +- ROADMAP.md | 2 +- docs/academic/06-multi-omics-integration.md | 8 +-- .../adr/0010-multi-omics-interfaces.md | 6 +- docs/audits/v0.3.0-baseline.md | 8 +++ src/openlongevity/analysis/omics.py | 19 +++++- src/openlongevity/analysis/pathway.py | 45 +++++++++---- tests/test_analysis.py | 65 ++++++++++++++++++- 9 files changed, 133 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 985df31..fe429a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ Documentation tooling now counts prose separately from fenced code and diagrams, The abbreviated license file has been replaced with the official Apache License 2.0 text. Project attribution is recorded separately in NOTICE, and citation metadata identifies the author as an independent Romanian researcher. The existing release tag is preserved. The citation's release version is not silently advanced to an unpublished development version merely because the Python package declares that version. -Known implementation issues remain visible: incomplete pathway adjustment, duplicate multi-omics handling, and the absence of an operational scientific review service. The frontend now exposes key provenance boundaries and has build-plus-browser smoke coverage, but it still needs broader accessibility, visual regression, and production deployment evidence. Publication origin classification is improved, but provider origin remains a retrieval-boundary label rather than scientific validation. No clinical validation or independent benchmark is asserted by this documentation entry. +Two analysis-integrity issues have targeted corrections after the documentation baseline. Pathway enrichment now applies monotonic Benjamini–Hochberg adjustment over the full valid pathway family, including valid pathways with no observed overlap in the correction denominator. Multi-omics grouping now rejects duplicate sample-layer pairs and inconsistent participant mappings instead of silently overwriting earlier values. The frontend now exposes key provenance boundaries and has build-plus-browser smoke coverage, but it still needs broader accessibility, visual regression, and production deployment evidence. Publication origin classification is improved, but provider origin remains a retrieval-boundary label rather than scientific validation. No clinical validation, independent benchmark, or operational scientific review service is asserted by this documentation entry. ## [0.2.0] - 2026-09-14 diff --git a/README.md b/README.md index e3ac460..222a742 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ The A–G taxonomy assigns systematic reviews to A, randomized trials to B, clin The numerical navigation score uses hand-selected design weights, confidence, replication metadata, sample size, and publication age. It is not a probability of truth, a treatment effect, or a validated evidence-certainty scale. Date-sensitive outputs can change with execution time. The [whitepaper](WHITEPAPER.md) now describes the actual arithmetic and its limitations. A research report should show the inputs and discuss whether conclusions depend on the ranking choices. -Experimental analysis utilities need similarly narrow interpretation. The biological-age routine is an ordinary-least-squares baseline. The survival utility produces Kaplan–Meier points without a complete inference framework. Multi-omics integration groups sample keys but does not establish participant consistency or statistical batch correction. The pathway routine has an identified multiple-testing adjustment limitation. These are reasons to inspect and improve the methods before drawing scientific conclusions, not details to hide behind a general research-use label. +Experimental analysis utilities need similarly narrow interpretation. The biological-age routine is an ordinary-least-squares baseline. The survival utility produces Kaplan–Meier points without a complete inference framework. Multi-omics integration now rejects duplicate sample-layer pairs and inconsistent participant mappings, but it still does not perform feature harmonization, statistical batch correction, or biological interpretation. The pathway routine now applies a monotonic Benjamini–Hochberg correction across the valid pathway hypothesis family, while results still depend on the declared universe, pathway annotations, and upstream gene selection. These are reasons to inspect methods before drawing scientific conclusions, not details to hide behind a general research-use label. ## Documentation, evidence, and contribution standards diff --git a/ROADMAP.md b/ROADMAP.md index 6ad37c1..8dd01cf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -38,7 +38,7 @@ Publication-origin classification has moved from a fixed false synthetic flag to Complete the documentation expansion with distinct, source-backed explanations rather than repeated filler. Every tracked Markdown file has a minimum prose target, but accuracy and implementation alignment remain separate requirements. The automated inventory should continue to expose short documents, missing attribution, and structural problems. Code examples need execution checks, and diagrams need to identify proposed components clearly. A passing word-count gate is not a substitute for editorial review. -Resolve known analysis limitations through focused changes and reference cases. The pathway adjustment needs a clearly defined hypothesis family and correct adjustment behavior. Multi-omics integration needs an explicit duplicate and participant-consistency policy. Date-sensitive scoring needs a controllable or clearly recorded time basis. Each change should state its effect on existing outputs and should not be promoted as clinical validation merely because a regression test passes. +Continue resolving analysis limitations through focused changes and reference cases. Pathway enrichment now defines the tested family as pathways with at least one member in the supplied universe and applies monotonic Benjamini–Hochberg adjustment while returning only observed overlaps. Multi-omics integration now rejects duplicate sample-layer pairs and inconsistent participant mappings instead of silently overwriting values. Remaining analysis priorities include a controllable or clearly recorded time basis for date-sensitive scoring, explicit manifests for preprocessing metadata, broader edge-case fixtures, and downstream methods that keep their scientific assumptions visible. Each change should state its effect on existing outputs and should not be promoted as clinical validation merely because a regression test passes. ## Publication-path acceptance diff --git a/docs/academic/06-multi-omics-integration.md b/docs/academic/06-multi-omics-integration.md index 348bba1..d9e8f0e 100644 --- a/docs/academic/06-multi-omics-integration.md +++ b/docs/academic/06-multi-omics-integration.md @@ -10,7 +10,7 @@ At baseline `9fddcbb`, `MultiOmicsSample` includes sample identifier, participan A sample identifier should refer to a defined physical or analytical sample, while a participant identifier links samples from the same person or experimental subject. These identifiers are not interchangeable. Multiple samples can arise from one participant at different times or from different tissues. A join using participant identity alone could combine measurements that were never intended to represent the same sampling occasion. Conversely, globally reusing a short sample label can merge unrelated studies accidentally. -The current implementation does not verify that repeated sample identifiers have consistent participant identifiers. It also overwrites an earlier feature dictionary when a later record has the same sample and layer. Those behaviors are concrete limitations, not theoretical concerns to be hidden in a generic warning. Before using the helper for cohort analysis, callers need to validate identity consistency and define a duplicate policy. A future implementation should reject ambiguous input or require an explicit, recorded resolution. +The current implementation now verifies two identity conditions at the grouping boundary. A sample identifier must map to one participant identifier, and a sample-layer pair may appear only once. If either condition is violated, the helper raises an error instead of returning a polished table that hides the ambiguity. This is a narrow but important improvement: it prevents participant mix-ups and repeated exports from becoming silent overwrites. It does not yet solve the harder scientific tasks of feature harmonization, batch correction, longitudinal alignment, or cohort-level missingness modeling. An integration manifest should record source dataset, participant namespace, sample namespace, collection time, tissue, assay layer, and the identifier mapping used. If privacy requirements prevent public release of participant identifiers, the reproducibility package still needs a controlled mapping strategy appropriate to the authorized setting. Public code does not justify exposing linkage keys. Synthetic identifiers should be used in repository examples and clearly distinguished from real cohort information. @@ -48,15 +48,15 @@ assert "transcriptomics" not in joined["S1"] ## Evaluation and failure cases -The acceptance suite should include an empty identifier, an inconsistent participant mapping, a repeated sample-layer pair, an explicitly missing value, an absent layer, and incompatible feature definitions. Separate tests of current behavior from tests of proposed safeguards. For example, a test documenting overwrite behavior should not be described as proving duplicate safety. A safeguard is complete only when the implementation rejects or resolves the ambiguity as specified. +The acceptance suite should include an empty identifier, an inconsistent participant mapping, a repeated sample-layer pair, an explicitly missing value, an absent layer, and incompatible feature definitions. The implemented safeguard now rejects inconsistent participant mappings and repeated sample-layer pairs. That should be described precisely: the helper prevents two high-risk identity failures, but it does not certify that all features are comparable or that a downstream model is scientifically appropriate. -Order sensitivity is particularly informative. Permuting distinct sample-layer inputs should preserve the same logical output, while permutations containing duplicates currently can change the result. Demonstrating that difference makes the limitation concrete. Once a duplicate policy is implemented, the test should show either a stable declared resolution or a consistent error. The project should preserve the original inputs so that an analyst can inspect which records participated in the join. +Order sensitivity is particularly informative. Permuting distinct sample-layer inputs should preserve the same logical output, while permutations containing duplicates now produce a consistent error instead of a different overwrite result. This is the correct behavior for the present interface because it forces a curator or upstream importer to decide whether the duplicate is an accidental repeat, a conflicting measurement, or a distinct sample that needs a different identifier. The project should preserve the original inputs so that an analyst can inspect which records participated in the join. Before downstream modeling, publish a summary of input samples, retained samples, missing layers, excluded records, and unresolved identifier issues. These counts describe data handling, not biological performance. A model report should link back to the exact integration manifest and transformation versions. The source reference for the present behavior is [`omics.py`](../../src/openlongevity/analysis/omics.py); [A09](09-model-evaluation.md) develops the separate evaluation boundary. **Question.** How can layers be joined without silently imputing absent measurements? -The integration interface uses a sample key and a typed layer enum. Explicit missing feature values remain `None`; absent layers remain absent. An integration manifest and duplicate safeguards are proposed additions, not current automatic output. +The integration interface uses a sample key and a typed layer enum. Explicit missing feature values remain `None`; absent layers remain absent. Duplicate and participant-consistency safeguards are now implemented at the grouping boundary. A richer integration manifest, feature namespaces, assay units, and preprocessing lineage remain proposed additions rather than current automatic output. ```mermaid flowchart TB diff --git a/docs/architecture/adr/0010-multi-omics-interfaces.md b/docs/architecture/adr/0010-multi-omics-interfaces.md index 424bd42..77357ba 100644 --- a/docs/architecture/adr/0010-multi-omics-interfaces.md +++ b/docs/architecture/adr/0010-multi-omics-interfaces.md @@ -39,7 +39,7 @@ The cost is that integration is postponed. The interface does not by itself solv ## Metadata Requirements -Each omics record should preserve sample identifier, participant identifier when available, layer, features, units or feature namespace where possible, batch, normalization method, source dataset, retrieval or creation date, and synthetic status where applicable. If the same sample appears in multiple layers, the grouping function should maintain layer separation. Duplicate sample-layer pairs should have a documented conflict policy. +Each omics record should preserve sample identifier, participant identifier when available, layer, features, units or feature namespace where possible, batch, normalization method, source dataset, retrieval or creation date, and synthetic status where applicable. If the same sample appears in multiple layers, the grouping function should maintain layer separation. The current conflict policy rejects duplicate sample-layer pairs and rejects sample identifiers that map to conflicting participant identifiers. Batch and normalization metadata are essential. Many omics signals are sensitive to platform, reagent lot, sequencing depth, laboratory site, and preprocessing pipeline. A value without preprocessing context can be difficult to compare. OpenLongevity should not overstate cross-study comparability when normalization methods differ. @@ -57,7 +57,7 @@ The platform's open-science ambition does not override participant protection. A ## Verification -Tests should confirm grouping behavior, preservation of layer identity, explicit missing-layer representation, and avoidance of silent imputation. Fixtures should include complete and incomplete samples, multiple layers, duplicate attempts, and different normalization labels. Documentation examples should state whether data are synthetic or real and should avoid clinical interpretation from raw multi-omics values. +Tests should confirm grouping behavior, preservation of layer identity, explicit missing-layer representation, and avoidance of silent imputation. Fixtures should include complete and incomplete samples, multiple layers, duplicate attempts, inconsistent participant linkage, and different normalization labels. The current suite verifies missing values, multiple layers for one sample, duplicate rejection, and participant-consistency rejection. Documentation examples should state whether data are synthetic or real and should avoid clinical interpretation from raw multi-omics values. This ADR remains accepted because it gives OpenLongevity a disciplined foundation for future multi-omics work. The project can connect layers without erasing their differences, and it can delay complex integration until methods and governance are ready. @@ -71,7 +71,7 @@ The acceptance criterion is that any multi-omics output can answer three questio ## Operational Risks -The operational risks are duplicate sample identifiers, inconsistent participant linkage, incompatible feature names, undocumented batch correction, and accidental exposure of sensitive individual-level records. The interface should force those issues into metadata rather than hiding them. A future ingestion workflow should reject ambiguous duplicates, record normalization state, and require explicit governance approval before handling real individual-level omics data. +The operational risks are duplicate sample identifiers, inconsistent participant linkage, incompatible feature names, undocumented batch correction, and accidental exposure of sensitive individual-level records. The interface now rejects two of those risks at grouping time: duplicate sample-layer pairs and conflicting participant linkage. The remaining risks still need metadata, ingestion manifests, feature namespaces, normalization records, and explicit governance approval before handling real individual-level omics data. The interface should also support negative capability statements. If a dataset lacks proteomics, longitudinal follow-up, or participant linkage, the grouped view should say so directly. Clear absence is better than a smooth table that suggests completeness. diff --git a/docs/audits/v0.3.0-baseline.md b/docs/audits/v0.3.0-baseline.md index 91f2a7a..5d27fd4 100644 --- a/docs/audits/v0.3.0-baseline.md +++ b/docs/audits/v0.3.0-baseline.md @@ -87,6 +87,14 @@ The verification added for this correction covers origin normalization rules, Pu This correction improves source interpretation, but it does not complete the scientific evidence workflow. `origin: provider` means the record crossed an implemented retrieval boundary; it does not mean the paper is correct, complete, unretracted, clinically relevant, or human reviewed. The product remains an alpha research prototype until frontend rendering, provider-backed evidence extraction, review operations, export manifests, and release gates are demonstrated together. +## Analysis-integrity follow-up: 23 September 2026 + +Two analysis utilities now have narrower and more defensible contracts than they had at the documentation baseline. Pathway enrichment defines its multiple-testing family as every pathway that has at least one member inside the supplied universe. Pathways with no observed overlap contribute to the Benjamini–Hochberg denominator, and the adjusted values are computed with the monotonic step-up rule rather than a simple rank multiplier. The result list remains focused on pathways with observed overlap, so callers do not receive a long list of empty hits, but the correction is no longer determined only by the pathways that happened to overlap. + +The multi-omics grouping helper now rejects duplicate sample-layer pairs and rejects a sample identifier that maps to more than one participant identifier. This removes the previous silent-overwrite behavior. The utility still returns a compact layer map for each sample and preserves explicit missing feature values as `None`. It does not perform batch correction, feature harmonization, unit conversion, privacy review, or biological integration. Those remain downstream scientific tasks. + +Verification for this follow-up added targeted regression cases for the pathway hypothesis family, monotonic adjusted p-values, genes outside the universe, missing omics values, duplicate omics layers, and inconsistent participant mappings. The local focused check reported `ruff check` passing and the analysis test module passing with seven tests. This is implementation evidence for the stated software contracts, not evidence that any pathway result or multi-omics association is biologically valid. + This audit records project responsibility under Ciprian Ștefan Pleșca, an independent Romanian researcher. It does not claim an external audit institution, university affiliation, scientific peer review, or security certification. Its contribution is a traceable account of what was inspected, what changed, what was actually checked, and what remains unresolved. Future updates should preserve that separation so that readers can evaluate progress without having to infer completion from presentation quality. --- diff --git a/src/openlongevity/analysis/omics.py b/src/openlongevity/analysis/omics.py index 27d3710..c405842 100644 --- a/src/openlongevity/analysis/omics.py +++ b/src/openlongevity/analysis/omics.py @@ -26,10 +26,25 @@ class MultiOmicsSample: def integrate_samples( samples: list[MultiOmicsSample], ) -> dict[str, dict[str, dict[str, float | None]]]: - """Group layers by common sample ID; missing values stay explicit as ``None``.""" + """Group layers by common sample ID; missing values stay explicit as ``None``. + + A sample identifier must map to a single participant, and each sample/layer pair + may appear only once. Silent overwrites are rejected because they can hide + participant mix-ups, repeated exports, or inconsistent upstream normalization. + """ result: dict[str, dict[str, dict[str, float | None]]] = {} + participants_by_sample: dict[str, str] = {} for sample in samples: if not sample.sample_id or not sample.participant_id: raise ValueError("sample_id and participant_id are required") - result.setdefault(sample.sample_id, {})[sample.layer.value] = dict(sample.features) + previous_participant = participants_by_sample.setdefault( + sample.sample_id, sample.participant_id + ) + if previous_participant != sample.participant_id: + raise ValueError("sample_id must map to exactly one participant_id") + + layers = result.setdefault(sample.sample_id, {}) + if sample.layer.value in layers: + raise ValueError("duplicate sample_id and layer combination") + layers[sample.layer.value] = dict(sample.features) return result diff --git a/src/openlongevity/analysis/pathway.py b/src/openlongevity/analysis/pathway.py index 44d2c8a..cd3b6a2 100644 --- a/src/openlongevity/analysis/pathway.py +++ b/src/openlongevity/analysis/pathway.py @@ -16,6 +16,14 @@ class EnrichmentResult: def enrich_gene_set( genes: set[str], pathways: dict[str, set[str]], universe: set[str] ) -> list[EnrichmentResult]: + """Compute one-sided pathway enrichment with monotonic Benjamini-Hochberg q-values. + + The multiple-testing family is every pathway that has at least one member inside the + supplied universe. Pathways with no observed overlap still contribute to the + correction factor, because excluding them would make adjusted p-values depend on + the observed result rather than the declared hypothesis family. The returned list + remains focused on pathways with at least one overlapping gene. + """ if not genes or not pathways or not universe or not genes <= universe: raise ValueError("genes must be non-empty and contained in universe") tests: list[tuple[str, int, int, float]] = [] @@ -24,17 +32,32 @@ def enrich_gene_set( for name, members in pathways.items(): overlap = len(genes & members) size = len(members & universe) - if not size or not overlap: + if not size: continue - tail = sum( - comb(size, k) * comb(population_size - size, sample_size - k) - for k in range(overlap, min(size, sample_size) + 1) - if sample_size - k <= population_size - size - ) - denominator = comb(population_size, sample_size) - tests.append((name, overlap, size, min(1.0, tail / denominator))) - tests.sort(key=lambda item: item[3]) + if overlap: + tail = sum( + comb(size, k) * comb(population_size - size, sample_size - k) + for k in range(overlap, min(size, sample_size) + 1) + if sample_size - k <= population_size - size + ) + denominator = comb(population_size, sample_size) + p_value = min(1.0, tail / denominator) + else: + p_value = 1.0 + tests.append((name, overlap, size, p_value)) + tests.sort(key=lambda item: (item[3], item[0])) + + adjusted_by_pathway: dict[str, float] = {} + running_adjusted = 1.0 + family_size = len(tests) + for rank, (name, _overlap, _size, p_value) in reversed( + list(enumerate(tests, start=1)) + ): + running_adjusted = min(running_adjusted, min(1.0, p_value * family_size / rank)) + adjusted_by_pathway[name] = running_adjusted + return [ - EnrichmentResult(name, overlap, size, p, min(1.0, p * len(tests) / rank)) - for rank, (name, overlap, size, p) in enumerate(tests, 1) + EnrichmentResult(name, overlap, size, p, adjusted_by_pathway[name]) + for name, overlap, size, p in tests + if overlap ] diff --git a/tests/test_analysis.py b/tests/test_analysis.py index b51915c..aa1e0f1 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -19,5 +19,68 @@ def test_kaplan_meier_handles_censoring() -> None: def test_pathway_enrichment_and_omics_join() -> None: result = enrich_gene_set({"A", "B"}, {"pathway": {"A", "B", "C"}}, {"A", "B", "C", "D"}) assert result[0].overlap == 2 - joined = integrate_samples([MultiOmicsSample("S1", "P1", OmicsLayer.GENOMICS, {"A": 1.0})]) + joined = integrate_samples( + [ + MultiOmicsSample("S1", "P1", OmicsLayer.GENOMICS, {"A": 1.0}), + MultiOmicsSample("S1", "P1", OmicsLayer.TRANSCRIPTOMICS, {"B": None}), + ] + ) assert joined["S1"]["genomics"]["A"] == 1.0 + assert joined["S1"]["transcriptomics"]["B"] is None + + +def test_pathway_enrichment_uses_full_valid_family_for_bh_adjustment() -> None: + results = enrich_gene_set( + {"A", "B"}, + { + "direct": {"A", "B"}, + "partial": {"A", "C"}, + "no_observed_overlap": {"E", "F"}, + "outside_universe": {"X", "Y"}, + }, + {"A", "B", "C", "D", "E", "F"}, + ) + + assert [result.pathway for result in results] == ["direct", "partial"] + assert results[0].p_value == 1 / 15 + assert results[0].adjusted_p_value == 0.2 + assert [result.adjusted_p_value for result in results] == sorted( + result.adjusted_p_value for result in results + ) + + +def test_pathway_enrichment_rejects_genes_outside_universe() -> None: + try: + enrich_gene_set({"A", "X"}, {"pathway": {"A"}}, {"A", "B"}) + except ValueError as exc: + assert "genes must be non-empty" in str(exc) + else: + raise AssertionError("Expected genes outside universe to be rejected") + + +def test_multi_omics_rejects_duplicate_sample_layer_pairs() -> None: + samples = [ + MultiOmicsSample("S1", "P1", OmicsLayer.GENOMICS, {"A": 1.0}), + MultiOmicsSample("S1", "P1", OmicsLayer.GENOMICS, {"A": 2.0}), + ] + + try: + integrate_samples(samples) + except ValueError as exc: + assert "duplicate sample_id and layer" in str(exc) + else: + raise AssertionError("Expected duplicate sample/layer pairs to be rejected") + + +def test_multi_omics_rejects_inconsistent_participant_mapping() -> None: + samples = [ + MultiOmicsSample("S1", "P1", OmicsLayer.GENOMICS, {"A": 1.0}), + MultiOmicsSample("S1", "P2", OmicsLayer.TRANSCRIPTOMICS, {"B": 2.0}), + ] + + try: + integrate_samples(samples) + except ValueError as exc: + assert "one participant_id" in str(exc) + else: + raise AssertionError("Expected inconsistent sample participant mapping to fail") From f7bb2a0270f89f9babcddde70dcd71ee2273cb7d Mon Sep 17 00:00:00 2001 From: Ciprian-LocalPulse Date: Wed, 23 Sep 2026 14:41:02 +0300 Subject: [PATCH 2/2] fix: make evidence scoring reproducible --- CHANGELOG.md | 2 +- README.md | 2 +- ROADMAP.md | 2 +- docs/API.md | 2 +- docs/audits/v0.3.0-baseline.md | 2 +- src/openlongevity/evidence.py | 35 ++++++++++++++++++++++++++-------- tests/test_core.py | 22 +++++++++++++++++++++ 7 files changed, 54 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe429a8..0f40105 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ Documentation tooling now counts prose separately from fenced code and diagrams, The abbreviated license file has been replaced with the official Apache License 2.0 text. Project attribution is recorded separately in NOTICE, and citation metadata identifies the author as an independent Romanian researcher. The existing release tag is preserved. The citation's release version is not silently advanced to an unpublished development version merely because the Python package declares that version. -Two analysis-integrity issues have targeted corrections after the documentation baseline. Pathway enrichment now applies monotonic Benjamini–Hochberg adjustment over the full valid pathway family, including valid pathways with no observed overlap in the correction denominator. Multi-omics grouping now rejects duplicate sample-layer pairs and inconsistent participant mappings instead of silently overwriting earlier values. The frontend now exposes key provenance boundaries and has build-plus-browser smoke coverage, but it still needs broader accessibility, visual regression, and production deployment evidence. Publication origin classification is improved, but provider origin remains a retrieval-boundary label rather than scientific validation. No clinical validation, independent benchmark, or operational scientific review service is asserted by this documentation entry. +Analysis-integrity work now includes reproducible time handling for navigation scores, plus the two targeted corrections recorded after the documentation baseline. Pathway enrichment now applies monotonic Benjamini–Hochberg adjustment over the full valid pathway family, including valid pathways with no observed overlap in the correction denominator. Multi-omics grouping now rejects duplicate sample-layer pairs and inconsistent participant mappings instead of silently overwriting earlier values. The frontend now exposes key provenance boundaries and has build-plus-browser smoke coverage, but it still needs broader accessibility, visual regression, and production deployment evidence. Publication origin classification is improved, but provider origin remains a retrieval-boundary label rather than scientific validation. No clinical validation, independent benchmark, or operational scientific review service is asserted by this documentation entry. ## [0.2.0] - 2026-09-14 diff --git a/README.md b/README.md index 222a742..a477ce7 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Only then consider a structured evidence observation. Define the study design, s The A–G taxonomy assigns systematic reviews to A, randomized trials to B, clinical studies to C, observational studies to D, animal studies to E, in-vitro studies to F, and computational work to G. This is the project's navigation convention. It is not a claim that every review is stronger than every experiment or a substitute for evaluating bias and relevance. The implementation also maps retracted records to G, so clients must retain original study design alongside publication status. -The numerical navigation score uses hand-selected design weights, confidence, replication metadata, sample size, and publication age. It is not a probability of truth, a treatment effect, or a validated evidence-certainty scale. Date-sensitive outputs can change with execution time. The [whitepaper](WHITEPAPER.md) now describes the actual arithmetic and its limitations. A research report should show the inputs and discuss whether conclusions depend on the ranking choices. +The numerical navigation score uses hand-selected design weights, confidence, replication metadata, sample size, and publication age. It is not a probability of truth, a treatment effect, or a validated evidence-certainty scale. Date-sensitive outputs can now be frozen by passing an explicit scoring time to the evidence engine; API summaries also expose `scoring_as_of` so exported interpretations can record the temporal basis. The [whitepaper](WHITEPAPER.md) describes the actual arithmetic and its limitations. A research report should show the inputs and discuss whether conclusions depend on the ranking choices. Experimental analysis utilities need similarly narrow interpretation. The biological-age routine is an ordinary-least-squares baseline. The survival utility produces Kaplan–Meier points without a complete inference framework. Multi-omics integration now rejects duplicate sample-layer pairs and inconsistent participant mappings, but it still does not perform feature harmonization, statistical batch correction, or biological interpretation. The pathway routine now applies a monotonic Benjamini–Hochberg correction across the valid pathway hypothesis family, while results still depend on the declared universe, pathway annotations, and upstream gene selection. These are reasons to inspect methods before drawing scientific conclusions, not details to hide behind a general research-use label. diff --git a/ROADMAP.md b/ROADMAP.md index 8dd01cf..c7e18da 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -38,7 +38,7 @@ Publication-origin classification has moved from a fixed false synthetic flag to Complete the documentation expansion with distinct, source-backed explanations rather than repeated filler. Every tracked Markdown file has a minimum prose target, but accuracy and implementation alignment remain separate requirements. The automated inventory should continue to expose short documents, missing attribution, and structural problems. Code examples need execution checks, and diagrams need to identify proposed components clearly. A passing word-count gate is not a substitute for editorial review. -Continue resolving analysis limitations through focused changes and reference cases. Pathway enrichment now defines the tested family as pathways with at least one member in the supplied universe and applies monotonic Benjamini–Hochberg adjustment while returning only observed overlaps. Multi-omics integration now rejects duplicate sample-layer pairs and inconsistent participant mappings instead of silently overwriting values. Remaining analysis priorities include a controllable or clearly recorded time basis for date-sensitive scoring, explicit manifests for preprocessing metadata, broader edge-case fixtures, and downstream methods that keep their scientific assumptions visible. Each change should state its effect on existing outputs and should not be promoted as clinical validation merely because a regression test passes. +Continue resolving analysis limitations through focused changes and reference cases. Pathway enrichment now defines the tested family as pathways with at least one member in the supplied universe and applies monotonic Benjamini–Hochberg adjustment while returning only observed overlaps. Multi-omics integration now rejects duplicate sample-layer pairs and inconsistent participant mappings instead of silently overwriting values. Evidence scoring now accepts an explicit `as_of` time and exposes the summary timestamp, reducing one reproducibility risk in date-sensitive navigation scores. Remaining analysis priorities include explicit manifests for preprocessing metadata, broader edge-case fixtures, and downstream methods that keep their scientific assumptions visible. Each change should state its effect on existing outputs and should not be promoted as clinical validation merely because a regression test passes. ## Publication-path acceptance diff --git a/docs/API.md b/docs/API.md index 0ceeb91..5feb1c5 100644 --- a/docs/API.md +++ b/docs/API.md @@ -85,7 +85,7 @@ The citation export route should not be described as a complete publication expo `GET /api/v1/evidence/{record_id}/review-events` lists stored review events for a fixture evidence record when the review repository is configured. The route returns audit events, not a full reviewer user interface. It is the persistence boundary for the human-review workflow: reviewer actions can be stored, inspected, and connected to citation-export eligibility, while user management and role delegation remain future work. -Evidence grades and scores require their methodological labels. The A–G mapping is a project taxonomy, and the numerical navigation score uses heuristic constants. Neither is a calibrated scientific certainty estimate. The score also depends on execution time when a publication date is present. The API's ability to serialize a number does not justify describing it as a treatment effect, probability of truth, or measure of human longevity benefit. +Evidence grades and scores require their methodological labels. The A–G mapping is a project taxonomy, and the numerical navigation score uses heuristic constants. Neither is a calibrated scientific certainty estimate. When a publication date is present, the score depends on the scoring time; evidence summaries now expose `scoring_as_of` so clients can record that temporal basis. The API's ability to serialize a number does not justify describing it as a treatment effect, probability of truth, or measure of human longevity benefit. Publication origin classification is now explicit for persisted publications, including synthetic seed rows and records saved through the PubMed ingestion path. Clients and operators must still avoid treating `synthetic: false` as proof of scientific reliability. It means the record was not classified as synthetic by the storage contract and entered through a provider-boundary path; it does not mean the publication is complete, unretracted, clinically relevant, or human reviewed. diff --git a/docs/audits/v0.3.0-baseline.md b/docs/audits/v0.3.0-baseline.md index 5d27fd4..76337c8 100644 --- a/docs/audits/v0.3.0-baseline.md +++ b/docs/audits/v0.3.0-baseline.md @@ -93,7 +93,7 @@ Two analysis utilities now have narrower and more defensible contracts than they The multi-omics grouping helper now rejects duplicate sample-layer pairs and rejects a sample identifier that maps to more than one participant identifier. This removes the previous silent-overwrite behavior. The utility still returns a compact layer map for each sample and preserves explicit missing feature values as `None`. It does not perform batch correction, feature harmonization, unit conversion, privacy review, or biological integration. Those remain downstream scientific tasks. -Verification for this follow-up added targeted regression cases for the pathway hypothesis family, monotonic adjusted p-values, genes outside the universe, missing omics values, duplicate omics layers, and inconsistent participant mappings. The local focused check reported `ruff check` passing and the analysis test module passing with seven tests. This is implementation evidence for the stated software contracts, not evidence that any pathway result or multi-omics association is biologically valid. +Verification for this follow-up added targeted regression cases for the pathway hypothesis family, monotonic adjusted p-values, genes outside the universe, missing omics values, duplicate omics layers, and inconsistent participant mappings. The local focused check reported `ruff check` passing and the analysis test module passing with seven tests. This is implementation evidence for the stated software contracts, not evidence that any pathway result or multi-omics association is biologically valid. A subsequent scoring reproducibility check adds an explicit `as_of` parameter for navigation-score age calculations and records `scoring_as_of` in evidence summaries, reducing drift caused by executing the same report on different dates. This audit records project responsibility under Ciprian Ștefan Pleșca, an independent Romanian researcher. It does not claim an external audit institution, university affiliation, scientific peer review, or security certification. Its contribution is a traceable account of what was inspected, what changed, what was actually checked, and what remains unresolved. Future updates should preserve that separation so that readers can evaluate progress without having to infer completion from presentation quality. diff --git a/src/openlongevity/evidence.py b/src/openlongevity/evidence.py index 55b9ad5..0ff4fc2 100644 --- a/src/openlongevity/evidence.py +++ b/src/openlongevity/evidence.py @@ -45,6 +45,17 @@ class EvidenceLevel(StrEnum): } +def _utc_datetime(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=UTC) + return value.astimezone(UTC) + + +def _parse_publication_datetime(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return _utc_datetime(parsed) + + class EvidenceEngine: """Grade and summarize records without implying clinical effectiveness.""" @@ -53,8 +64,13 @@ def grade(self, record: EvidenceRecord) -> EvidenceLevel: return EvidenceLevel.G return _LEVEL_BY_TYPE[record.study_type] - def score(self, record: EvidenceRecord) -> float: - """Return a transparent navigation score, not a validated effect estimate.""" + def score(self, record: EvidenceRecord, *, as_of: datetime | None = None) -> float: + """Return a transparent navigation score, not a validated effect estimate. + + ``as_of`` freezes the publication-age component for reproducible reports. When + omitted, the current UTC time preserves the historical runtime behavior. + """ + scoring_time = _utc_datetime(as_of) if as_of else datetime.now(UTC) score = _BASE_SCORE[record.study_type] * record.confidence if record.replication_status.casefold() in {"replicated", "independent"}: score *= 1.15 @@ -68,10 +84,7 @@ def score(self, record: EvidenceRecord) -> float: try: age_years = max( 0.0, - ( - datetime.now(UTC) - - datetime.fromisoformat(record.publication_date).replace(tzinfo=UTC) - ).days + (scoring_time - _parse_publication_datetime(record.publication_date)).days / 365.25, ) score *= max(0.75, 1.0 - age_years * 0.01) @@ -79,9 +92,12 @@ def score(self, record: EvidenceRecord) -> float: pass return round(min(1.0, max(0.0, score)), 4) - def summarize(self, records: Iterable[EvidenceRecord]) -> dict[str, object]: + def summarize( + self, records: Iterable[EvidenceRecord], *, as_of: datetime | None = None + ) -> dict[str, object]: records = tuple(records) active = tuple(r for r in records if r.retraction_status is not RetractionStatus.RETRACTED) + scoring_time = _utc_datetime(as_of) if as_of else datetime.now(UTC) grades = Counter(self.grade(r).value for r in active) mean = sum(r.confidence for r in active) / len(active) if active else 0.0 return { @@ -89,9 +105,12 @@ def summarize(self, records: Iterable[EvidenceRecord]) -> dict[str, object]: "active_records": len(active), "evidence_distribution": dict(sorted(grades.items())), "mean_confidence": round(mean, 4), - "mean_navigation_score": round(sum(self.score(r) for r in active) / len(active), 4) + "mean_navigation_score": round( + sum(self.score(r, as_of=scoring_time) for r in active) / len(active), 4 + ) if active else 0.0, + "scoring_as_of": scoring_time.isoformat(), "disclaimer": "Research use only. Not medical advice.", } diff --git a/tests/test_core.py b/tests/test_core.py index 96687ed..69ce346 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,3 +1,5 @@ +from datetime import UTC, datetime + from openlongevity.evidence import EvidenceEngine, EvidenceLevel from openlongevity.gaps import ResearchGapDetector from openlongevity.graph import EvidenceGraph @@ -28,6 +30,26 @@ def test_grading_and_retraction() -> None: ) +def test_navigation_score_accepts_explicit_scoring_time() -> None: + engine = EvidenceEngine() + dated = record( + "dated", + StudyType.RCT, + publication_date="2020-01-01", + replication_status="replicated", + sample_size=200, + ) + early = datetime(2021, 1, 1, tzinfo=UTC) + later = datetime(2031, 1, 1, tzinfo=UTC) + + assert engine.score(dated, as_of=early) == engine.score(dated, as_of=early) + assert engine.score(dated, as_of=early) > engine.score(dated, as_of=later) + + summary = engine.summarize([dated], as_of=early) + assert summary["scoring_as_of"] == "2021-01-01T00:00:00+00:00" + assert summary["mean_navigation_score"] == engine.score(dated, as_of=early) + + def test_gap_detector_flags_translation_gap() -> None: gaps = ResearchGapDetector().detect("senescence", [record("a", StudyType.ANIMAL)]) assert gaps[0].kind == "translational_gap"